Skip to content

SDK / exoplayer

Go to a project
Commit 85d094a2 by olly Committed by Oliver Woodman
Options

Move surface frame-rate adjustment into the helper 

Estimating the playback frame-rate, querying the display refresh rate, and
setting the surface frame-rate, are all closely related to one another. In
particular because setting the surface frame-rate can directly cause the
display refresh rate to change. It therefore makes sense to move surface
frame-rate adjustment into the helper.

This also makes it easier to re-use the logic in other video renderers.

PiperOrigin-RevId: 348455864
parent 696bb34a
Inline Side-by-side
Showing with 144 additions and 118 deletions
library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java
......@@ -112,7 +112,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
private static boolean deviceNeedsSetOutputSurfaceWorkaround;
private final Context context;
private final VideoFrameReleaseTimeHelper frameReleaseTimeHelper;
private final VideoFrameReleaseHelper frameReleaseHelper;
private final EventDispatcher eventDispatcher;
private final long allowedJoiningTimeMs;
private final int maxDroppedFramesToNotify;
......@@ -123,7 +123,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
private boolean codecHandlesHdr10PlusOutOfBandMetadata;
@Nullable private Surface surface;
private float surfaceFrameRate;
@Nullable private Surface dummySurface;
private boolean haveReportedFirstFrameRenderedForCurrentSurface;
@C.VideoScalingMode private int scalingMode;
......@@ -278,7 +277,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
this.allowedJoiningTimeMs = allowedJoiningTimeMs;
this.maxDroppedFramesToNotify = maxDroppedFramesToNotify;
this.context = context.getApplicationContext();
frameReleaseTimeHelper = new VideoFrameReleaseTimeHelper(this.context);
frameReleaseHelper = new VideoFrameReleaseHelper(this.context);
eventDispatcher = new EventDispatcher(eventHandler, eventListener);
deviceNeedsNoPostProcessWorkaround = deviceNeedsNoPostProcessWorkaround();
joiningDeadlineMs = C.TIME_UNSET;
......@@ -408,7 +407,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
releaseCodec();
}
eventDispatcher.enabled(decoderCounters);
frameReleaseTimeHelper.onEnabled();
frameReleaseHelper.onEnabled();
mayRenderFirstFrameAfterEnableIfNotStarted = mayRenderStartOfStream;
renderedFirstFrameAfterEnable = false;
}
......@@ -417,7 +416,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
protected void onPositionReset(long positionUs, boolean joining) throws ExoPlaybackException {
super.onPositionReset(positionUs, joining);
clearRenderedFirstFrame();
frameReleaseTimeHelper.onPositionReset();
frameReleaseHelper.onPositionReset();
lastBufferPresentationTimeUs = C.TIME_UNSET;
initialPositionUs = C.TIME_UNSET;
consecutiveDroppedFrameCount = 0;
......@@ -459,8 +458,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
lastRenderRealtimeUs = SystemClock.elapsedRealtime() * 1000;
totalVideoFrameProcessingOffsetUs = 0;
videoFrameProcessingOffsetCount = 0;
frameReleaseTimeHelper.onStarted();
updateSurfaceFrameRate(/* isNewSurface= */ false);
frameReleaseHelper.onStarted();
}
@Override
......@@ -468,7 +466,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
joiningDeadlineMs = C.TIME_UNSET;
maybeNotifyDroppedFrames();
maybeNotifyVideoFrameProcessingOffset();
clearSurfaceFrameRate();
frameReleaseHelper.onStopped();
super.onStopped();
}
......@@ -477,7 +475,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
clearReportedVideoSize();
clearRenderedFirstFrame();
haveReportedFirstFrameRenderedForCurrentSurface = false;
frameReleaseTimeHelper.onDisabled();
frameReleaseHelper.onDisabled();
tunnelingOnFrameRenderedListener = null;
try {
super.onDisabled();
......@@ -533,10 +531,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
}
// We only need to update the codec if the surface has changed.
if (this.surface != surface) {
clearSurfaceFrameRate();
this.surface = surface;
frameReleaseHelper.onSurfaceChanged(surface);
haveReportedFirstFrameRenderedForCurrentSurface = false;
updateSurfaceFrameRate(/* isNewSurface= */ true);
@State int state = getState();
@Nullable MediaCodecAdapter codec = getCodec();
......@@ -643,8 +640,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
@Override
public void setPlaybackSpeed(float playbackSpeed) throws ExoPlaybackException {
super.setPlaybackSpeed(playbackSpeed);
frameReleaseTimeHelper.onPlaybackSpeed(playbackSpeed);
updateSurfaceFrameRate(/* isNewSurface= */ false);
frameReleaseHelper.onPlaybackSpeed(playbackSpeed);
}
@Override
......@@ -749,8 +745,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
// On API level 20 and below the decoder does not apply the rotation.
currentUnappliedRotationDegrees = format.rotationDegrees;
}
frameReleaseTimeHelper.onFormatChanged(format.frameRate);
updateSurfaceFrameRate(/* isNewSurface= */ false);
frameReleaseHelper.onFormatChanged(format.frameRate);
}
@Override
......@@ -805,7 +800,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
}
if (bufferPresentationTimeUs != lastBufferPresentationTimeUs) {
frameReleaseTimeHelper.onNextFrame(bufferPresentationTimeUs);
frameReleaseHelper.onNextFrame(bufferPresentationTimeUs);
this.lastBufferPresentationTimeUs = bufferPresentationTimeUs;
}
......@@ -873,8 +868,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
long unadjustedFrameReleaseTimeNs = systemTimeNs + (earlyUs * 1000);
// Apply a timestamp adjustment, if there is one.
long adjustedReleaseTimeNs =
frameReleaseTimeHelper.adjustReleaseTime(unadjustedFrameReleaseTimeNs);
long adjustedReleaseTimeNs = frameReleaseHelper.adjustReleaseTime(unadjustedFrameReleaseTimeNs);
earlyUs = (adjustedReleaseTimeNs - systemTimeNs) / 1000;
boolean treatDroppedBuffersAsSkipped = joiningDeadlineMs != C.TIME_UNSET;
......@@ -1133,50 +1127,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
maybeNotifyRenderedFirstFrame();
}
/**
* Updates the frame-rate of the current {@link #surface} based on the renderer operating rate,
* frame-rate of the content, and whether the renderer is started.
*
* @param isNewSurface Whether the current {@link #surface} is new.
*/
private void updateSurfaceFrameRate(boolean isNewSurface) {
if (Util.SDK_INT < 30 || surface == null || surface == dummySurface) {
return;
}
float playbackFrameRate = frameReleaseTimeHelper.getPlaybackFrameRate();
float surfaceFrameRate =
getState() == STATE_STARTED && playbackFrameRate != C.RATE_UNSET ? playbackFrameRate : 0;
// We always set the frame-rate if we have a new surface, since we have no way of knowing what
// it might have been set to previously.
if (this.surfaceFrameRate == surfaceFrameRate && !isNewSurface) {
return;
}
this.surfaceFrameRate = surfaceFrameRate;
setSurfaceFrameRateV30(surface, surfaceFrameRate);
}
/** Clears the frame-rate of the current {@link #surface}. */
private void clearSurfaceFrameRate() {
if (Util.SDK_INT < 30 || surface == null || surface == dummySurface || surfaceFrameRate == 0) {
return;
}
surfaceFrameRate = 0;
setSurfaceFrameRateV30(surface, /* frameRate= */ 0);
}
@RequiresApi(30)
private static void setSurfaceFrameRateV30(Surface surface, float frameRate) {
int compatibility =
frameRate == 0
? Surface.FRAME_RATE_COMPATIBILITY_DEFAULT
: Surface.FRAME_RATE_COMPATIBILITY_FIXED_SOURCE;
try {
surface.setFrameRate(frameRate, compatibility);
} catch (IllegalStateException e) {
Log.e(TAG, "Failed to call Surface.setFrameRate", e);
}
}
private boolean shouldUseDummySurface(MediaCodecInfo codecInfo) {
return Util.SDK_INT >= 23
&& !tunneling
......
library/core/src/main/java/com/google/android/exoplayer2/video/VideoFrameReleaseTimeHelper.java → library/core/src/main/java/com/google/android/exoplayer2/video/VideoFrameReleaseHelper.java
......@@ -15,6 +15,8 @@
*/
package com.google.android.exoplayer2.video;
import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import android.annotation.TargetApi;
import android.content.Context;
import android.hardware.display.DisplayManager;
......@@ -24,6 +26,7 @@ import android.os.Message;
import android.view.Choreographer;
import android.view.Choreographer.FrameCallback;
import android.view.Display;
import android.view.Surface;
import android.view.WindowManager;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
......@@ -32,14 +35,21 @@ import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.Renderer;
import com.google.android.exoplayer2.util.Log;
import com.google.android.exoplayer2.util.Util;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* Makes a best effort to adjust frame release timestamps for a video {@link Renderer} in order to
* achieve a smoother visual result.
* Helps a video {@link Renderer} release frames to a {@link Surface}. The helper:
*
* <ul>
* <li>Adjusts frame release timestamps to achieve a smoother visual result. The release
* timestamps are smoothed, and aligned with the default display's vsync signal.
* <li>Adjusts the {@link Surface} frame rate to inform the underlying platform of a fixed frame
* rate, when there is one.
* </ul>
*/
public final class VideoFrameReleaseTimeHelper {
public final class VideoFrameReleaseHelper {
private static final String TAG = "VideoFrameReleaseTimeHelper";
private static final String TAG = "VideoFrameReleaseHelper";
/** The period between sampling display VSYNC timestamps, in milliseconds. */
private static final long VSYNC_SAMPLE_UPDATE_PERIOD_MS = 500;
......@@ -60,6 +70,10 @@ public final class VideoFrameReleaseTimeHelper {
@Nullable private final VSyncSampler vsyncSampler;
@Nullable private final DefaultDisplayListener displayListener;
private boolean started;
@Nullable private Surface surface;
private float surfaceFrameRate;
private float formatFrameRate;
private double playbackSpeed;
......@@ -73,20 +87,11 @@ public final class VideoFrameReleaseTimeHelper {
private long lastAdjustedReleaseTimeNs;
/**
* Constructs an instance that smooths frame release timestamps but does not align them with
* the default display's vsync signal.
*/
public VideoFrameReleaseTimeHelper() {
this(null);
}
/**
* Constructs an instance that smooths frame release timestamps and aligns them with the default
* display's vsync signal.
* Constructs an instance.
*
* @param context A context from which information about the default display can be retrieved.
*/
public VideoFrameReleaseTimeHelper(@Nullable Context context) {
public VideoFrameReleaseHelper(@Nullable Context context) {
fixedFrameRateEstimator = new FixedFrameRateEstimator();
if (context != null) {
context = context.getApplicationContext();
......@@ -95,7 +100,8 @@ public final class VideoFrameReleaseTimeHelper {
windowManager = null;
}
if (windowManager != null) {
displayListener = Util.SDK_INT >= 17 ? maybeBuildDefaultDisplayListenerV17(context) : null;
displayListener =
Util.SDK_INT >= 17 ? maybeBuildDefaultDisplayListenerV17(checkNotNull(context)) : null;
vsyncSampler = VSyncSampler.getInstance();
} else {
displayListener = null;
......@@ -112,7 +118,7 @@ public final class VideoFrameReleaseTimeHelper {
public void onEnabled() {
fixedFrameRateEstimator.reset();
if (windowManager != null) {
vsyncSampler.addObserver();
checkNotNull(vsyncSampler).addObserver();
if (displayListener != null) {
displayListener.register();
}
......@@ -120,20 +126,29 @@ public final class VideoFrameReleaseTimeHelper {
}
}
/** Called when the renderer is disabled. */
@TargetApi(17) // displayListener is null if Util.SDK_INT < 17.
public void onDisabled() {
if (windowManager != null) {
if (displayListener != null) {
displayListener.unregister();
}
vsyncSampler.removeObserver();
}
}
/** Called when the renderer is started. */
public void onStarted() {
started = true;
resetAdjustment();
updateSurfaceFrameRate(/* isNewSurface= */ false);
}
/**
* Called when the renderer changes which {@link Surface} it's rendering to renders to.
*
* @param surface The new {@link Surface}, or {@code null} if the renderer does not have one.
*/
public void onSurfaceChanged(@Nullable Surface surface) {
if (surface instanceof DummySurface) {
// We don't care about dummy surfaces for release timing, since they're not visible.
surface = null;
}
if (this.surface == surface) {
return;
}
clearSurfaceFrameRate();
this.surface = surface;
updateSurfaceFrameRate(/* isNewSurface= */ true);
}
/** Called when the renderer's position is reset. */
......@@ -150,6 +165,7 @@ public final class VideoFrameReleaseTimeHelper {
public void onPlaybackSpeed(double playbackSpeed) {
this.playbackSpeed = playbackSpeed;
resetAdjustment();
updateSurfaceFrameRate(/* isNewSurface= */ false);
}
/**
......@@ -160,6 +176,7 @@ public final class VideoFrameReleaseTimeHelper {
public void onFormatChanged(float formatFrameRate) {
this.formatFrameRate = formatFrameRate;
fixedFrameRateEstimator.onFormatChanged(formatFrameRate);
updateSurfaceFrameRate(/* isNewSurface= */ false);
}
/**
......@@ -176,14 +193,25 @@ public final class VideoFrameReleaseTimeHelper {
frameIndex++;
}
/** Returns the estimated playback frame rate, or {@link C#RATE_UNSET} if unknown. */
public float getPlaybackFrameRate() {
// TODO: Hook up fixedFrameRateEstimator.
return formatFrameRate == Format.NO_VALUE
? C.RATE_UNSET
: (float) (formatFrameRate * playbackSpeed);
/** Called when the renderer is stopped. */
public void onStopped() {
started = false;
clearSurfaceFrameRate();
}
/** Called when the renderer is disabled. */
@TargetApi(17) // displayListener is null if Util.SDK_INT < 17.
public void onDisabled() {
if (windowManager != null) {
if (displayListener != null) {
displayListener.unregister();
}
checkNotNull(vsyncSampler).removeObserver();
}
}
// Frame release time adjustment.
/**
* Adjusts the release timestamp for the next frame. This is the frame whose presentation
* timestamp was most recently passed to {@link #onNextFrame}.
......@@ -206,7 +234,7 @@ public final class VideoFrameReleaseTimeHelper {
long frameDurationNs = fixedFrameRateEstimator.getFrameDurationNs();
long candidateAdjustedReleaseTimeNs =
lastAdjustedReleaseTimeNs
+ getPlayoutDuration(frameDurationNs * (frameIndex - lastAdjustedFrameIndex));
+ (long) ((frameDurationNs * (frameIndex - lastAdjustedFrameIndex)) / playbackSpeed);
if (adjustmentAllowed(releaseTimeNs, candidateAdjustedReleaseTimeNs)) {
adjustedReleaseTimeNs = candidateAdjustedReleaseTimeNs;
} else {
......@@ -235,14 +263,71 @@ public final class VideoFrameReleaseTimeHelper {
pendingLastAdjustedFrameIndex = C.INDEX_UNSET;
}
private static boolean adjustmentAllowed(
long unadjustedReleaseTimeNs, long adjustedReleaseTimeNs) {
return Math.abs(unadjustedReleaseTimeNs - adjustedReleaseTimeNs) <= MAX_ALLOWED_ADJUSTMENT_NS;
}
// Surface frame rate adjustment.
/**
* Updates the frame-rate of the current {@link #surface} based on the renderer operating rate,
* frame-rate of the content, and whether the renderer is started.
*
* @param isNewSurface Whether the current {@link #surface} is new.
*/
private void updateSurfaceFrameRate(boolean isNewSurface) {
if (Util.SDK_INT < 30 || surface == null) {
return;
}
float surfaceFrameRate = 0;
// TODO: Hook up fixedFrameRateEstimator.
if (started && formatFrameRate != Format.NO_VALUE) {
surfaceFrameRate = (float) (formatFrameRate * playbackSpeed);
}
// We always set the frame-rate if we have a new surface, since we have no way of knowing what
// it might have been set to previously.
if (this.surfaceFrameRate == surfaceFrameRate && !isNewSurface) {
return;
}
this.surfaceFrameRate = surfaceFrameRate;
setSurfaceFrameRateV30(surface, surfaceFrameRate);
}
/** Clears the frame-rate of the current {@link #surface}. */
private void clearSurfaceFrameRate() {
if (Util.SDK_INT < 30 || surface == null || surfaceFrameRate == 0) {
return;
}
surfaceFrameRate = 0;
setSurfaceFrameRateV30(surface, /* frameRate= */ 0);
}
@RequiresApi(30)
private static void setSurfaceFrameRateV30(Surface surface, float frameRate) {
int compatibility =
frameRate == 0
? Surface.FRAME_RATE_COMPATIBILITY_DEFAULT
: Surface.FRAME_RATE_COMPATIBILITY_FIXED_SOURCE;
try {
surface.setFrameRate(frameRate, compatibility);
} catch (IllegalStateException e) {
Log.e(TAG, "Failed to call Surface.setFrameRate", e);
}
}
// Display refresh rate and vsync logic.
@RequiresApi(17)
@Nullable
private DefaultDisplayListener maybeBuildDefaultDisplayListenerV17(Context context) {
DisplayManager manager = (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE);
return manager == null ? null : new DefaultDisplayListener(manager);
}
private void updateDefaultDisplayRefreshRateParams() {
Display defaultDisplay = windowManager.getDefaultDisplay();
Display defaultDisplay = checkNotNull(windowManager).getDefaultDisplay();
if (defaultDisplay != null) {
double defaultDisplayRefreshRate = defaultDisplay.getRefreshRate();
vsyncDurationNs = (long) (C.NANOS_PER_SECOND / defaultDisplayRefreshRate);
......@@ -254,15 +339,6 @@ public final class VideoFrameReleaseTimeHelper {
}
}
private long getPlayoutDuration(long mediaDuration) {
return (long) (mediaDuration / playbackSpeed);
}
private static boolean adjustmentAllowed(
long unadjustedReleaseTimeNs, long adjustedReleaseTimeNs) {
return Math.abs(unadjustedReleaseTimeNs - adjustedReleaseTimeNs) <= MAX_ALLOWED_ADJUSTMENT_NS;
}
private static long closestVsync(long releaseTime, long sampledVsyncTime, long vsyncDuration) {
long vsyncCount = (releaseTime - sampledVsyncTime) / vsyncDuration;
long snappedTimeNs = sampledVsyncTime + (vsyncDuration * vsyncCount);
......@@ -290,7 +366,7 @@ public final class VideoFrameReleaseTimeHelper {
}
public void register() {
displayManager.registerDisplayListener(this, null);
displayManager.registerDisplayListener(this, Util.createHandlerForCurrentLooper());
}
public void unregister() {
......@@ -318,8 +394,8 @@ public final class VideoFrameReleaseTimeHelper {
/**
* Samples display vsync timestamps. A single instance using a single {@link Choreographer} is
* shared by all {@link VideoFrameReleaseTimeHelper} instances. This is done to avoid a resource
* leak in the platform on API levels prior to 23. See [Internal: b/12455729].
* shared by all {@link VideoFrameReleaseHelper} instances. This is done to avoid a resource leak
* in the platform on API levels prior to 23. See [Internal: b/12455729].
*/
private static final class VSyncSampler implements FrameCallback, Handler.Callback {
......@@ -333,7 +409,7 @@ public final class VideoFrameReleaseTimeHelper {
private final Handler handler;
private final HandlerThread choreographerOwnerThread;
private Choreographer choreographer;
@MonotonicNonNull private Choreographer choreographer;
private int observerCount;
public static VSyncSampler getInstance() {
......@@ -349,16 +425,16 @@ public final class VideoFrameReleaseTimeHelper {
}
/**
* Notifies the sampler that a {@link VideoFrameReleaseTimeHelper} is observing
* {@link #sampledVsyncTimeNs}, and hence that the value should be periodically updated.
* Notifies the sampler that a {@link VideoFrameReleaseHelper} is observing {@link
* #sampledVsyncTimeNs}, and hence that the value should be periodically updated.
*/
public void addObserver() {
handler.sendEmptyMessage(MSG_ADD_OBSERVER);
}
/**
* Notifies the sampler that a {@link VideoFrameReleaseTimeHelper} is no longer observing
* {@link #sampledVsyncTimeNs}.
* Notifies the sampler that a {@link VideoFrameReleaseHelper} is no longer observing {@link
* #sampledVsyncTimeNs}.
*/
public void removeObserver() {
handler.sendEmptyMessage(MSG_REMOVE_OBSERVER);
......@@ -367,7 +443,7 @@ public final class VideoFrameReleaseTimeHelper {
@Override
public void doFrame(long vsyncTimeNs) {
sampledVsyncTimeNs = vsyncTimeNs;
choreographer.postFrameCallbackDelayed(this, VSYNC_SAMPLE_UPDATE_PERIOD_MS);
checkNotNull(choreographer).postFrameCallbackDelayed(this, VSYNC_SAMPLE_UPDATE_PERIOD_MS);
}
@Override
......@@ -398,14 +474,14 @@ public final class VideoFrameReleaseTimeHelper {
private void addObserverInternal() {
observerCount++;
if (observerCount == 1) {
choreographer.postFrameCallback(this);
checkNotNull(choreographer).postFrameCallback(this);
}
}
private void removeObserverInternal() {
observerCount--;
if (observerCount == 0) {
choreographer.removeFrameCallback(this);
checkNotNull(choreographer).removeFrameCallback(this);
sampledVsyncTimeNs = C.TIME_UNSET;
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment