Skip to content

SDK / exoplayer

Go to a project
Commit d0165438 by olly Committed by Ian Baker
Options

VideoFrameReleaseTimeHelper: Split out frame-rate estimation 

PiperOrigin-RevId: 346554044
parent f18d81f8
Expand all
Inline Side-by-side
Showing with 250 additions and 50 deletions
library/core/src/main/java/com/google/android/exoplayer2/video/FixedFrameRateEstimator.java 0 → 100644
/*
* Copyright (C) 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.video;
import androidx.annotation.VisibleForTesting;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.Format;
import java.util.Arrays;
/**
* Attempts to detect and refine a fixed frame rate estimate based on frame presentation timestamps.
*/
/* package */ final class FixedFrameRateEstimator {
/**
* The number of consecutive matching frame durations for the tracker to be considered in sync.
*/
@VisibleForTesting static final int CONSECUTIVE_MATCHING_FRAME_DURATIONS_FOR_SYNC = 10;
/**
* The maximum amount frame durations can differ for them to be considered matching, in
* nanoseconds.
*
* <p>This constant is set to 1ms to account for container formats that only represent frame
* presentation timestamps to the nearest millisecond. In such cases, frame durations need to
* switch between values that are 1ms apart to achieve common fixed frame rates (e.g., 30fps
* content will need frames that are 33ms and 34ms).
*/
@VisibleForTesting static final long MAX_MATCHING_FRAME_DIFFERENCE_NS = 1_000_000;
private Matcher currentMatcher;
private Matcher candidateMatcher;
private boolean candidateMatcherActive;
private boolean switchToCandidateMatcherWhenSynced;
private float formatFrameRate;
private long lastFramePresentationTimeNs;
public FixedFrameRateEstimator() {
currentMatcher = new Matcher();
candidateMatcher = new Matcher();
formatFrameRate = Format.NO_VALUE;
lastFramePresentationTimeNs = C.TIME_UNSET;
}
/** Resets the estimator. */
public void reset() {
currentMatcher.reset();
candidateMatcher.reset();
candidateMatcherActive = false;
formatFrameRate = Format.NO_VALUE;
lastFramePresentationTimeNs = C.TIME_UNSET;
}
/**
* Called when the renderer's output format changes.
*
* @param formatFrameRate The format's frame rate, or {@link Format#NO_VALUE} if unknown.
*/
public void onFormatChanged(float formatFrameRate) {
// The format frame rate is only used to determine to what extent the estimator should be reset.
// Frame rate estimates are always calculated directly from frame presentation timestamps.
if (this.formatFrameRate != formatFrameRate) {
reset();
} else {
// Keep the current matcher, but prefer to switch to a new matcher once synced even if the
// current one does not lose sync. This avoids an issue where the current matcher would
// continue to be used if a frame rate change has occurred that's too small to trigger sync
// loss (e.g., a change from 30fps to 29.97fps) and which is not represented in the format
// frame rates (e.g., because they're unset or only have integer precision).
switchToCandidateMatcherWhenSynced = true;
}
this.formatFrameRate = formatFrameRate;
}
/**
* Called with each frame presentation timestamp.
*
* @param framePresentationTimeNs The frame presentation timestamp, in nanoseconds.
*/
public void onNextFrame(long framePresentationTimeNs) {
currentMatcher.onNextFrame(framePresentationTimeNs);
if (currentMatcher.isSynced() && !switchToCandidateMatcherWhenSynced) {
candidateMatcherActive = false;
} else if (lastFramePresentationTimeNs != C.TIME_UNSET) {
if (!candidateMatcherActive || candidateMatcher.isLastFrameOutlier()) {
// Reset the candidate with the last and current frame presentation timestamps, so that it
// will try and match against the duration of the previous frame.
candidateMatcher.reset();
candidateMatcher.onNextFrame(lastFramePresentationTimeNs);
}
candidateMatcherActive = true;
candidateMatcher.onNextFrame(framePresentationTimeNs);
}
if (candidateMatcherActive && candidateMatcher.isSynced()) {
// The candidate matcher should be promoted to be the current matcher. The current matcher
// can be re-used as the next candidate matcher.
Matcher previousMatcher = currentMatcher;
currentMatcher = candidateMatcher;
candidateMatcher = previousMatcher;
candidateMatcherActive = false;
switchToCandidateMatcherWhenSynced = false;
}
lastFramePresentationTimeNs = framePresentationTimeNs;
}
/** Returns whether the estimator has detected a fixed frame rate. */
public boolean isSynced() {
return currentMatcher.isSynced();
}
/**
* The currently detected fixed frame duration estimate in nanoseconds, or {@link C#TIME_UNSET} if
* {@link #isSynced()} is {@code false}. Whilst synced, the estimate is refined each time {@link
* #onNextFrame} is called with a new frame presentation timestamp.
*/
public long getFrameDurationNs() {
return isSynced() ? currentMatcher.getFrameDurationNs() : C.TIME_UNSET;
}
/**
* The currently detected fixed frame rate estimate, or {@link Format#NO_VALUE} if {@link
* #isSynced()} is {@code false}. Whilst synced, the estimate is refined each time {@link
* #onNextFrame} is called with a new frame presentation timestamp.
*/
public double getFrameRate() {
return isSynced()
? (double) C.NANOS_PER_SECOND / currentMatcher.getFrameDurationNs()
: Format.NO_VALUE;
}
/** Tries to match frame durations against the duration of the first frame it receives. */
private static final class Matcher {
private long firstFramePresentationTimeNs;
private long firstFrameDurationNs;
private long lastFramePresentationTimeNs;
private long frameCount;
/** The total number of frames that have matched the frame duration being tracked. */
private long matchingFrameCount;
/** The sum of the frame durations of all matching frames. */
private long matchingFrameDurationSumNs;
/** Cyclic buffer of flags indicating whether the most recent frame durations were outliers. */
private final boolean[] recentFrameOutlierFlags;
/**
* The number of recent frame durations that were outliers. Equal to the number of {@code true}
* values in {@link #recentFrameOutlierFlags}.
*/
private int recentFrameOutlierCount;
public Matcher() {
recentFrameOutlierFlags = new boolean[CONSECUTIVE_MATCHING_FRAME_DURATIONS_FOR_SYNC];
}
public void reset() {
frameCount = 0;
matchingFrameCount = 0;
matchingFrameDurationSumNs = 0;
recentFrameOutlierCount = 0;
Arrays.fill(recentFrameOutlierFlags, false);
}
public boolean isSynced() {
return frameCount > CONSECUTIVE_MATCHING_FRAME_DURATIONS_FOR_SYNC
&& recentFrameOutlierCount == 0;
}
public boolean isLastFrameOutlier() {
if (frameCount == 0) {
return false;
}
return recentFrameOutlierFlags[getRecentFrameOutlierIndex(frameCount - 1)];
}
public long getFrameDurationNs() {
return matchingFrameCount == 0 ? 0 : (matchingFrameDurationSumNs / matchingFrameCount);
}
public void onNextFrame(long framePresentationTimeNs) {
if (frameCount == 0) {
firstFramePresentationTimeNs = framePresentationTimeNs;
} else if (frameCount == 1) {
// This is the frame duration that the tracker will match against.
firstFrameDurationNs = framePresentationTimeNs - firstFramePresentationTimeNs;
matchingFrameDurationSumNs = firstFrameDurationNs;
matchingFrameCount = 1;
} else {
long lastFrameDurationNs = framePresentationTimeNs - lastFramePresentationTimeNs;
int recentFrameOutlierIndex = getRecentFrameOutlierIndex(frameCount);
if (Math.abs(lastFrameDurationNs - firstFrameDurationNs)
<= MAX_MATCHING_FRAME_DIFFERENCE_NS) {
matchingFrameCount++;
matchingFrameDurationSumNs += lastFrameDurationNs;
if (recentFrameOutlierFlags[recentFrameOutlierIndex]) {
recentFrameOutlierFlags[recentFrameOutlierIndex] = false;
recentFrameOutlierCount--;
}
} else {
if (!recentFrameOutlierFlags[recentFrameOutlierIndex]) {
recentFrameOutlierFlags[recentFrameOutlierIndex] = true;
recentFrameOutlierCount++;
}
}
}
frameCount++;
lastFramePresentationTimeNs = framePresentationTimeNs;
}
private static int getRecentFrameOutlierIndex(long frameCount) {
return (int) (frameCount % CONSECUTIVE_MATCHING_FRAME_DURATIONS_FOR_SYNC);
}
}
}
library/core/src/main/java/com/google/android/exoplayer2/video/VideoFrameReleaseTimeHelper.java
...@@ -55,24 +55,18 @@ public final class VideoFrameReleaseTimeHelper { ...@@ -55,24 +55,18 @@ public final class VideoFrameReleaseTimeHelper {
*/ */
private static final long VSYNC_OFFSET_PERCENTAGE = 80; private static final long VSYNC_OFFSET_PERCENTAGE = 80;
private static final int MIN_FRAMES_FOR_ADJUSTMENT = 6; private final FixedFrameRateEstimator fixedFrameRateEstimator;
@Nullable private final WindowManager windowManager; @Nullable private final WindowManager windowManager;
@Nullable private final VSyncSampler vsyncSampler; @Nullable private final VSyncSampler vsyncSampler;
@Nullable private final DefaultDisplayListener displayListener; @Nullable private final DefaultDisplayListener displayListener;
private float formatFrameRate; private float formatFrameRate;
private double playbackSpeed; private double playbackSpeed;
private long nextFramePresentationTimeUs;
private long vsyncDurationNs; private long vsyncDurationNs;
private long vsyncOffsetNs; private long vsyncOffsetNs;
private boolean haveSync; private long frameIndex;
private long syncReleaseTimeNs;
private long syncFramePresentationTimeNs;
private long frameCount;
private long pendingLastAdjustedFrameIndex; private long pendingLastAdjustedFrameIndex;
private long pendingLastAdjustedReleaseTimeNs; private long pendingLastAdjustedReleaseTimeNs;
private long lastAdjustedFrameIndex; private long lastAdjustedFrameIndex;
...@@ -93,6 +87,7 @@ public final class VideoFrameReleaseTimeHelper { ...@@ -93,6 +87,7 @@ public final class VideoFrameReleaseTimeHelper {
* @param context A context from which information about the default display can be retrieved. * @param context A context from which information about the default display can be retrieved.
*/ */
public VideoFrameReleaseTimeHelper(@Nullable Context context) { public VideoFrameReleaseTimeHelper(@Nullable Context context) {
fixedFrameRateEstimator = new FixedFrameRateEstimator();
if (context != null) { if (context != null) {
context = context.getApplicationContext(); context = context.getApplicationContext();
windowManager = (WindowManager) context.getSystemService(Context.WINDOW_SERVICE); windowManager = (WindowManager) context.getSystemService(Context.WINDOW_SERVICE);
...@@ -115,7 +110,7 @@ public final class VideoFrameReleaseTimeHelper { ...@@ -115,7 +110,7 @@ public final class VideoFrameReleaseTimeHelper {
/** Called when the renderer is enabled. */ /** Called when the renderer is enabled. */
@TargetApi(17) // displayListener is null if Util.SDK_INT < 17. @TargetApi(17) // displayListener is null if Util.SDK_INT < 17.
public void onEnabled() { public void onEnabled() {
haveSync = false; fixedFrameRateEstimator.reset();
if (windowManager != null) { if (windowManager != null) {
vsyncSampler.addObserver(); vsyncSampler.addObserver();
if (displayListener != null) { if (displayListener != null) {
...@@ -138,12 +133,12 @@ public final class VideoFrameReleaseTimeHelper { ...@@ -138,12 +133,12 @@ public final class VideoFrameReleaseTimeHelper {
/** Called when the renderer is started. */ /** Called when the renderer is started. */
public void onStarted() { public void onStarted() {
haveSync = false; resetAdjustment();
} }
/** Called when the renderer's position is reset. */ /** Called when the renderer's position is reset. */
public void onPositionReset() { public void onPositionReset() {
haveSync = false; resetAdjustment();
} }
/** /**
...@@ -154,7 +149,7 @@ public final class VideoFrameReleaseTimeHelper { ...@@ -154,7 +149,7 @@ public final class VideoFrameReleaseTimeHelper {
*/ */
public void onPlaybackSpeed(double playbackSpeed) { public void onPlaybackSpeed(double playbackSpeed) {
this.playbackSpeed = playbackSpeed; this.playbackSpeed = playbackSpeed;
haveSync = false; resetAdjustment();
} }
/** /**
...@@ -164,6 +159,7 @@ public final class VideoFrameReleaseTimeHelper { ...@@ -164,6 +159,7 @@ public final class VideoFrameReleaseTimeHelper {
*/ */
public void onFormatChanged(float formatFrameRate) { public void onFormatChanged(float formatFrameRate) {
this.formatFrameRate = formatFrameRate; this.formatFrameRate = formatFrameRate;
fixedFrameRateEstimator.onFormatChanged(formatFrameRate);
} }
/** /**
...@@ -172,14 +168,17 @@ public final class VideoFrameReleaseTimeHelper { ...@@ -172,14 +168,17 @@ public final class VideoFrameReleaseTimeHelper {
* @param framePresentationTimeUs The frame presentation timestamp, in microseconds. * @param framePresentationTimeUs The frame presentation timestamp, in microseconds.
*/ */
public void onNextFrame(long framePresentationTimeUs) { public void onNextFrame(long framePresentationTimeUs) {
if (pendingLastAdjustedFrameIndex != C.INDEX_UNSET) {
lastAdjustedFrameIndex = pendingLastAdjustedFrameIndex; lastAdjustedFrameIndex = pendingLastAdjustedFrameIndex;
lastAdjustedReleaseTimeNs = pendingLastAdjustedReleaseTimeNs; lastAdjustedReleaseTimeNs = pendingLastAdjustedReleaseTimeNs;
nextFramePresentationTimeUs = framePresentationTimeUs; }
frameCount++; fixedFrameRateEstimator.onNextFrame(framePresentationTimeUs * 1000);
frameIndex++;
} }
/** Returns the estimated playback frame rate, or {@link C#RATE_UNSET} if unknown. */ /** Returns the estimated playback frame rate, or {@link C#RATE_UNSET} if unknown. */
public float getPlaybackFrameRate() { public float getPlaybackFrameRate() {
// TODO: Hook up fixedFrameRateEstimator.
return formatFrameRate == Format.NO_VALUE return formatFrameRate == Format.NO_VALUE
? C.RATE_UNSET ? C.RATE_UNSET
: (float) (formatFrameRate * playbackSpeed); : (float) (formatFrameRate * playbackSpeed);
...@@ -200,51 +199,21 @@ public final class VideoFrameReleaseTimeHelper { ...@@ -200,51 +199,21 @@ public final class VideoFrameReleaseTimeHelper {
* {@link System#nanoTime()}. * {@link System#nanoTime()}.
*/ */
public long adjustReleaseTime(long releaseTimeNs) { public long adjustReleaseTime(long releaseTimeNs) {
long framePresentationTimeNs = nextFramePresentationTimeUs * 1000;
// Until we know better, the adjustment will be a no-op. // Until we know better, the adjustment will be a no-op.
long adjustedReleaseTimeNs = releaseTimeNs; long adjustedReleaseTimeNs = releaseTimeNs;
if (haveSync) { if (lastAdjustedFrameIndex != C.INDEX_UNSET && fixedFrameRateEstimator.isSynced()) {
if (frameCount >= MIN_FRAMES_FOR_ADJUSTMENT) { long frameDurationNs = fixedFrameRateEstimator.getFrameDurationNs();
// We're synced and have waited the required number of frames to apply an adjustment.
// Calculate the average frame time across all the frames we've seen since the last sync.
// This will typically give us a frame rate at a finer granularity than the frame times
// themselves (which often only have millisecond granularity).
long averageFrameDurationNs = (framePresentationTimeNs - syncFramePresentationTimeNs)
/ frameCount;
// Project the adjusted frame time forward using the average.
long candidateAdjustedReleaseTimeNs = long candidateAdjustedReleaseTimeNs =
lastAdjustedReleaseTimeNs lastAdjustedReleaseTimeNs
+ getPlayoutDuration( + getPlayoutDuration(frameDurationNs * (frameIndex - lastAdjustedFrameIndex));
averageFrameDurationNs * (frameCount - lastAdjustedFrameIndex));
if (adjustmentAllowed(releaseTimeNs, candidateAdjustedReleaseTimeNs)) { if (adjustmentAllowed(releaseTimeNs, candidateAdjustedReleaseTimeNs)) {
adjustedReleaseTimeNs = candidateAdjustedReleaseTimeNs; adjustedReleaseTimeNs = candidateAdjustedReleaseTimeNs;
} else { } else {
haveSync = false; resetAdjustment();
}
} else {
// We're synced but haven't waited the required number of frames to apply an adjustment.
// Check for drift between the proposed and projected frame release timestamps.
long projectedReleaseTimeNs =
syncReleaseTimeNs
+ getPlayoutDuration(framePresentationTimeNs - syncFramePresentationTimeNs);
if (!adjustmentAllowed(releaseTimeNs, projectedReleaseTimeNs)) {
haveSync = false;
} }
} }
} pendingLastAdjustedFrameIndex = frameIndex;
// If we need to sync, do so now.
if (!haveSync) {
syncFramePresentationTimeNs = framePresentationTimeNs;
syncReleaseTimeNs = releaseTimeNs;
frameCount = 0;
haveSync = true;
}
pendingLastAdjustedFrameIndex = frameCount;
pendingLastAdjustedReleaseTimeNs = adjustedReleaseTimeNs; pendingLastAdjustedReleaseTimeNs = adjustedReleaseTimeNs;
if (vsyncSampler == null || vsyncDurationNs == C.TIME_UNSET) { if (vsyncSampler == null || vsyncDurationNs == C.TIME_UNSET) {
...@@ -254,13 +223,18 @@ public final class VideoFrameReleaseTimeHelper { ...@@ -254,13 +223,18 @@ public final class VideoFrameReleaseTimeHelper {
if (sampledVsyncTimeNs == C.TIME_UNSET) { if (sampledVsyncTimeNs == C.TIME_UNSET) {
return adjustedReleaseTimeNs; return adjustedReleaseTimeNs;
} }
// Find the timestamp of the closest vsync. This is the vsync that we're targeting. // Find the timestamp of the closest vsync. This is the vsync that we're targeting.
long snappedTimeNs = closestVsync(adjustedReleaseTimeNs, sampledVsyncTimeNs, vsyncDurationNs); long snappedTimeNs = closestVsync(adjustedReleaseTimeNs, sampledVsyncTimeNs, vsyncDurationNs);
// Apply an offset so that we release before the target vsync, but after the previous one. // Apply an offset so that we release before the target vsync, but after the previous one.
return snappedTimeNs - vsyncOffsetNs; return snappedTimeNs - vsyncOffsetNs;
} }
private void resetAdjustment() {
frameIndex = 0;
lastAdjustedFrameIndex = C.INDEX_UNSET;
pendingLastAdjustedFrameIndex = C.INDEX_UNSET;
}
@RequiresApi(17) @RequiresApi(17)
private DefaultDisplayListener maybeBuildDefaultDisplayListenerV17(Context context) { private DefaultDisplayListener maybeBuildDefaultDisplayListenerV17(Context context) {
DisplayManager manager = (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE); DisplayManager manager = (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment