VideoFrameReleaseTimeHelper: Split out frame-rate estimation

PiperOrigin-RevId: 346554044

library/core/src/main/java/com/google/android/exoplayer2/video/FixedFrameRateEstimator.java

+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.video;
+
+import androidx.annotation.VisibleForTesting;
+import com.google.android.exoplayer2.C;
+import com.google.android.exoplayer2.Format;
+import java.util.Arrays;
+
+/**
+ * Attempts to detect and refine a fixed frame rate estimate based on frame presentation timestamps.
+ */
+/* package */ final class FixedFrameRateEstimator {
+
+  /**
+   * The number of consecutive matching frame durations for the tracker to be considered in sync.
+   */
+  @VisibleForTesting static final int CONSECUTIVE_MATCHING_FRAME_DURATIONS_FOR_SYNC = 10;
+  /**
+   * The maximum amount frame durations can differ for them to be considered matching, in
+   * nanoseconds.
+   *
+   * <p>This constant is set to 1ms to account for container formats that only represent frame
+   * presentation timestamps to the nearest millisecond. In such cases, frame durations need to
+   * switch between values that are 1ms apart to achieve common fixed frame rates (e.g., 30fps
+   * content will need frames that are 33ms and 34ms).
+   */
+  @VisibleForTesting static final long MAX_MATCHING_FRAME_DIFFERENCE_NS = 1_000_000;
+
+  private Matcher currentMatcher;
+  private Matcher candidateMatcher;
+  private boolean candidateMatcherActive;
+  private boolean switchToCandidateMatcherWhenSynced;
+  private float formatFrameRate;
+  private long lastFramePresentationTimeNs;
+
+  public FixedFrameRateEstimator() {
+    currentMatcher = new Matcher();
+    candidateMatcher = new Matcher();
+    formatFrameRate = Format.NO_VALUE;
+    lastFramePresentationTimeNs = C.TIME_UNSET;
+  }
+
+  /** Resets the estimator. */
+  public void reset() {
+    currentMatcher.reset();
+    candidateMatcher.reset();
+    candidateMatcherActive = false;
+    formatFrameRate = Format.NO_VALUE;
+    lastFramePresentationTimeNs = C.TIME_UNSET;
+  }
+
+  /**
+   * Called when the renderer's output format changes.
+   *
+   * @param formatFrameRate The format's frame rate, or {@link Format#NO_VALUE} if unknown.
+   */
+  public void onFormatChanged(float formatFrameRate) {
+    // The format frame rate is only used to determine to what extent the estimator should be reset.
+    // Frame rate estimates are always calculated directly from frame presentation timestamps.
+    if (this.formatFrameRate != formatFrameRate) {
+      reset();
+    } else {
+      // Keep the current matcher, but prefer to switch to a new matcher once synced even if the
+      // current one does not lose sync. This avoids an issue where the current matcher would
+      // continue to be used if a frame rate change has occurred that's too small to trigger sync
+      // loss (e.g., a change from 30fps to 29.97fps) and which is not represented in the format
+      // frame rates (e.g., because they're unset or only have integer precision).
+      switchToCandidateMatcherWhenSynced = true;
+    }
+    this.formatFrameRate = formatFrameRate;
+  }
+
+  /**
+   * Called with each frame presentation timestamp.
+   *
+   * @param framePresentationTimeNs The frame presentation timestamp, in nanoseconds.
+   */
+  public void onNextFrame(long framePresentationTimeNs) {
+    currentMatcher.onNextFrame(framePresentationTimeNs);
+    if (currentMatcher.isSynced() && !switchToCandidateMatcherWhenSynced) {
+      candidateMatcherActive = false;
+    } else if (lastFramePresentationTimeNs != C.TIME_UNSET) {
+      if (!candidateMatcherActive || candidateMatcher.isLastFrameOutlier()) {
+        // Reset the candidate with the last and current frame presentation timestamps, so that it
+        // will try and match against the duration of the previous frame.
+        candidateMatcher.reset();
+        candidateMatcher.onNextFrame(lastFramePresentationTimeNs);
+      }
+      candidateMatcherActive = true;
+      candidateMatcher.onNextFrame(framePresentationTimeNs);
+    }
+    if (candidateMatcherActive && candidateMatcher.isSynced()) {
+      // The candidate matcher should be promoted to be the current matcher. The current matcher
+      // can be re-used as the next candidate matcher.
+      Matcher previousMatcher = currentMatcher;
+      currentMatcher = candidateMatcher;
+      candidateMatcher = previousMatcher;
+      candidateMatcherActive = false;
+      switchToCandidateMatcherWhenSynced = false;
+    }
+    lastFramePresentationTimeNs = framePresentationTimeNs;
+  }
+
+  /** Returns whether the estimator has detected a fixed frame rate. */
+  public boolean isSynced() {
+    return currentMatcher.isSynced();
+  }
+
+  /**
+   * The currently detected fixed frame duration estimate in nanoseconds, or {@link C#TIME_UNSET} if
+   * {@link #isSynced()} is {@code false}. Whilst synced, the estimate is refined each time {@link
+   * #onNextFrame} is called with a new frame presentation timestamp.
+   */
+  public long getFrameDurationNs() {
+    return isSynced() ? currentMatcher.getFrameDurationNs() : C.TIME_UNSET;
+  }
+
+  /**
+   * The currently detected fixed frame rate estimate, or {@link Format#NO_VALUE} if {@link
+   * #isSynced()} is {@code false}. Whilst synced, the estimate is refined each time {@link
+   * #onNextFrame} is called with a new frame presentation timestamp.
+   */
+  public double getFrameRate() {
+    return isSynced()
+        ? (double) C.NANOS_PER_SECOND / currentMatcher.getFrameDurationNs()
+        : Format.NO_VALUE;
+  }
+
+  /** Tries to match frame durations against the duration of the first frame it receives. */
+  private static final class Matcher {
+
+    private long firstFramePresentationTimeNs;
+    private long firstFrameDurationNs;
+    private long lastFramePresentationTimeNs;
+    private long frameCount;
+
+    /** The total number of frames that have matched the frame duration being tracked. */
+    private long matchingFrameCount;
+    /** The sum of the frame durations of all matching frames. */
+    private long matchingFrameDurationSumNs;
+    /** Cyclic buffer of flags indicating whether the most recent frame durations were outliers. */
+    private final boolean[] recentFrameOutlierFlags;
+    /**
+     * The number of recent frame durations that were outliers. Equal to the number of {@code true}
+     * values in {@link #recentFrameOutlierFlags}.
+     */
+    private int recentFrameOutlierCount;
+
+    public Matcher() {
+      recentFrameOutlierFlags = new boolean[CONSECUTIVE_MATCHING_FRAME_DURATIONS_FOR_SYNC];
+    }
+
+    public void reset() {
+      frameCount = 0;
+      matchingFrameCount = 0;
+      matchingFrameDurationSumNs = 0;
+      recentFrameOutlierCount = 0;
+      Arrays.fill(recentFrameOutlierFlags, false);
+    }
+
+    public boolean isSynced() {
+      return frameCount > CONSECUTIVE_MATCHING_FRAME_DURATIONS_FOR_SYNC
+          && recentFrameOutlierCount == 0;
+    }
+
+    public boolean isLastFrameOutlier() {
+      if (frameCount == 0) {
+        return false;
+      }
+      return recentFrameOutlierFlags[getRecentFrameOutlierIndex(frameCount - 1)];
+    }
+
+    public long getFrameDurationNs() {
+      return matchingFrameCount == 0 ? 0 : (matchingFrameDurationSumNs / matchingFrameCount);
+    }
+
+    public void onNextFrame(long framePresentationTimeNs) {
+      if (frameCount == 0) {
+        firstFramePresentationTimeNs = framePresentationTimeNs;
+      } else if (frameCount == 1) {
+        // This is the frame duration that the tracker will match against.
+        firstFrameDurationNs = framePresentationTimeNs - firstFramePresentationTimeNs;
+        matchingFrameDurationSumNs = firstFrameDurationNs;
+        matchingFrameCount = 1;
+      } else {
+        long lastFrameDurationNs = framePresentationTimeNs - lastFramePresentationTimeNs;
+        int recentFrameOutlierIndex = getRecentFrameOutlierIndex(frameCount);
+        if (Math.abs(lastFrameDurationNs - firstFrameDurationNs)
+            <= MAX_MATCHING_FRAME_DIFFERENCE_NS) {
+          matchingFrameCount++;
+          matchingFrameDurationSumNs += lastFrameDurationNs;
+          if (recentFrameOutlierFlags[recentFrameOutlierIndex]) {
+            recentFrameOutlierFlags[recentFrameOutlierIndex] = false;
+            recentFrameOutlierCount--;
+          }
+        } else {
+          if (!recentFrameOutlierFlags[recentFrameOutlierIndex]) {
+            recentFrameOutlierFlags[recentFrameOutlierIndex] = true;
+            recentFrameOutlierCount++;
+          }
+        }
+      }
+
+      frameCount++;
+      lastFramePresentationTimeNs = framePresentationTimeNs;
+    }
+
+    private static int getRecentFrameOutlierIndex(long frameCount) {
+      return (int) (frameCount % CONSECUTIVE_MATCHING_FRAME_DURATIONS_FOR_SYNC);
+    }
+  }
+}

library/core/src/main/java/com/google/android/exoplayer2/video/VideoFrameReleaseTimeHelper.java

@@ -55,24 +55,18 @@ public final class VideoFrameReleaseTimeHelper {
    */
    */
   private static final long VSYNC_OFFSET_PERCENTAGE = 80;
   private static final long VSYNC_OFFSET_PERCENTAGE = 80;
 
 
-  private static final int MIN_FRAMES_FOR_ADJUSTMENT = 6;
-
+  private final FixedFrameRateEstimator fixedFrameRateEstimator;
   @Nullable private final WindowManager windowManager;
   @Nullable private final WindowManager windowManager;
   @Nullable private final VSyncSampler vsyncSampler;
   @Nullable private final VSyncSampler vsyncSampler;
   @Nullable private final DefaultDisplayListener displayListener;
   @Nullable private final DefaultDisplayListener displayListener;
 
 
   private float formatFrameRate;
   private float formatFrameRate;
   private double playbackSpeed;
   private double playbackSpeed;
-  private long nextFramePresentationTimeUs;
 
 
   private long vsyncDurationNs;
   private long vsyncDurationNs;
   private long vsyncOffsetNs;
   private long vsyncOffsetNs;
 
 
-  private boolean haveSync;
-  private long syncReleaseTimeNs;
-  private long syncFramePresentationTimeNs;
-  private long frameCount;
-
+  private long frameIndex;
   private long pendingLastAdjustedFrameIndex;
   private long pendingLastAdjustedFrameIndex;
   private long pendingLastAdjustedReleaseTimeNs;
   private long pendingLastAdjustedReleaseTimeNs;
   private long lastAdjustedFrameIndex;
   private long lastAdjustedFrameIndex;
@@ -93,6 +87,7 @@ public final class VideoFrameReleaseTimeHelper {
    * @param context A context from which information about the default display can be retrieved.
    * @param context A context from which information about the default display can be retrieved.
    */
    */
   public VideoFrameReleaseTimeHelper(@Nullable Context context) {
   public VideoFrameReleaseTimeHelper(@Nullable Context context) {
+    fixedFrameRateEstimator = new FixedFrameRateEstimator();
     if (context != null) {
     if (context != null) {
       context = context.getApplicationContext();
       context = context.getApplicationContext();
       windowManager = (WindowManager) context.getSystemService(Context.WINDOW_SERVICE);
       windowManager = (WindowManager) context.getSystemService(Context.WINDOW_SERVICE);
@@ -115,7 +110,7 @@ public final class VideoFrameReleaseTimeHelper {
   /** Called when the renderer is enabled. */
   /** Called when the renderer is enabled. */
   @TargetApi(17) // displayListener is null if Util.SDK_INT < 17.
   @TargetApi(17) // displayListener is null if Util.SDK_INT < 17.
   public void onEnabled() {
   public void onEnabled() {
-    haveSync = false;
+    fixedFrameRateEstimator.reset();
     if (windowManager != null) {
     if (windowManager != null) {
       vsyncSampler.addObserver();
       vsyncSampler.addObserver();
       if (displayListener != null) {
       if (displayListener != null) {
@@ -138,12 +133,12 @@ public final class VideoFrameReleaseTimeHelper {
 
 
   /** Called when the renderer is started. */
   /** Called when the renderer is started. */
   public void onStarted() {
   public void onStarted() {
-    haveSync = false;
+    resetAdjustment();
   }
   }
 
 
   /** Called when the renderer's position is reset. */
   /** Called when the renderer's position is reset. */
   public void onPositionReset() {
   public void onPositionReset() {
-    haveSync = false;
+    resetAdjustment();
   }
   }
 
 
   /**
   /**
@@ -154,7 +149,7 @@ public final class VideoFrameReleaseTimeHelper {
    */
    */
   public void onPlaybackSpeed(double playbackSpeed) {
   public void onPlaybackSpeed(double playbackSpeed) {
     this.playbackSpeed = playbackSpeed;
     this.playbackSpeed = playbackSpeed;
-    haveSync = false;
+    resetAdjustment();
   }
   }
 
 
   /**
   /**
@@ -164,6 +159,7 @@ public final class VideoFrameReleaseTimeHelper {
    */
    */
   public void onFormatChanged(float formatFrameRate) {
   public void onFormatChanged(float formatFrameRate) {
     this.formatFrameRate = formatFrameRate;
     this.formatFrameRate = formatFrameRate;
+    fixedFrameRateEstimator.onFormatChanged(formatFrameRate);
   }
   }
 
 
   /**
   /**
@@ -172,14 +168,17 @@ public final class VideoFrameReleaseTimeHelper {
    * @param framePresentationTimeUs The frame presentation timestamp, in microseconds.
    * @param framePresentationTimeUs The frame presentation timestamp, in microseconds.
    */
    */
   public void onNextFrame(long framePresentationTimeUs) {
   public void onNextFrame(long framePresentationTimeUs) {
-    lastAdjustedFrameIndex = pendingLastAdjustedFrameIndex;
-    lastAdjustedReleaseTimeNs = pendingLastAdjustedReleaseTimeNs;
-    nextFramePresentationTimeUs = framePresentationTimeUs;
-    frameCount++;
+    if (pendingLastAdjustedFrameIndex != C.INDEX_UNSET) {
+      lastAdjustedFrameIndex = pendingLastAdjustedFrameIndex;
+      lastAdjustedReleaseTimeNs = pendingLastAdjustedReleaseTimeNs;
+    }
+    fixedFrameRateEstimator.onNextFrame(framePresentationTimeUs * 1000);
+    frameIndex++;
   }
   }
 
 
   /** Returns the estimated playback frame rate, or {@link C#RATE_UNSET} if unknown. */
   /** Returns the estimated playback frame rate, or {@link C#RATE_UNSET} if unknown. */
   public float getPlaybackFrameRate() {
   public float getPlaybackFrameRate() {
+    // TODO: Hook up fixedFrameRateEstimator.
     return formatFrameRate == Format.NO_VALUE
     return formatFrameRate == Format.NO_VALUE
         ? C.RATE_UNSET
         ? C.RATE_UNSET
         : (float) (formatFrameRate * playbackSpeed);
         : (float) (formatFrameRate * playbackSpeed);
@@ -200,51 +199,21 @@ public final class VideoFrameReleaseTimeHelper {
    *     {@link System#nanoTime()}.
    *     {@link System#nanoTime()}.
    */
    */
   public long adjustReleaseTime(long releaseTimeNs) {
   public long adjustReleaseTime(long releaseTimeNs) {
-    long framePresentationTimeNs = nextFramePresentationTimeUs * 1000;
-
     // Until we know better, the adjustment will be a no-op.
     // Until we know better, the adjustment will be a no-op.
     long adjustedReleaseTimeNs = releaseTimeNs;
     long adjustedReleaseTimeNs = releaseTimeNs;
 
 
-    if (haveSync) {
-      if (frameCount >= MIN_FRAMES_FOR_ADJUSTMENT) {
-        // We're synced and have waited the required number of frames to apply an adjustment.
-        // Calculate the average frame time across all the frames we've seen since the last sync.
-        // This will typically give us a frame rate at a finer granularity than the frame times
-        // themselves (which often only have millisecond granularity).
-        long averageFrameDurationNs = (framePresentationTimeNs - syncFramePresentationTimeNs)
-            / frameCount;
-        // Project the adjusted frame time forward using the average.
-        long candidateAdjustedReleaseTimeNs =
-            lastAdjustedReleaseTimeNs
-                + getPlayoutDuration(
-                    averageFrameDurationNs * (frameCount - lastAdjustedFrameIndex));
-
-        if (adjustmentAllowed(releaseTimeNs, candidateAdjustedReleaseTimeNs)) {
-          adjustedReleaseTimeNs = candidateAdjustedReleaseTimeNs;
-        } else {
-          haveSync = false;
-        }
+    if (lastAdjustedFrameIndex != C.INDEX_UNSET && fixedFrameRateEstimator.isSynced()) {
+      long frameDurationNs = fixedFrameRateEstimator.getFrameDurationNs();
+      long candidateAdjustedReleaseTimeNs =
+          lastAdjustedReleaseTimeNs
+              + getPlayoutDuration(frameDurationNs * (frameIndex - lastAdjustedFrameIndex));
+      if (adjustmentAllowed(releaseTimeNs, candidateAdjustedReleaseTimeNs)) {
+        adjustedReleaseTimeNs = candidateAdjustedReleaseTimeNs;
       } else {
       } else {
-        // We're synced but haven't waited the required number of frames to apply an adjustment.
-        // Check for drift between the proposed and projected frame release timestamps.
-        long projectedReleaseTimeNs =
-            syncReleaseTimeNs
-                + getPlayoutDuration(framePresentationTimeNs - syncFramePresentationTimeNs);
-        if (!adjustmentAllowed(releaseTimeNs, projectedReleaseTimeNs)) {
-          haveSync = false;
-        }
+        resetAdjustment();
       }
       }
     }
     }
-
-    // If we need to sync, do so now.
-    if (!haveSync) {
-      syncFramePresentationTimeNs = framePresentationTimeNs;
-      syncReleaseTimeNs = releaseTimeNs;
-      frameCount = 0;
-      haveSync = true;
-    }
-
-    pendingLastAdjustedFrameIndex = frameCount;
+    pendingLastAdjustedFrameIndex = frameIndex;
     pendingLastAdjustedReleaseTimeNs = adjustedReleaseTimeNs;
     pendingLastAdjustedReleaseTimeNs = adjustedReleaseTimeNs;
 
 
     if (vsyncSampler == null || vsyncDurationNs == C.TIME_UNSET) {
     if (vsyncSampler == null || vsyncDurationNs == C.TIME_UNSET) {
@@ -254,13 +223,18 @@ public final class VideoFrameReleaseTimeHelper {
     if (sampledVsyncTimeNs == C.TIME_UNSET) {
     if (sampledVsyncTimeNs == C.TIME_UNSET) {
       return adjustedReleaseTimeNs;
       return adjustedReleaseTimeNs;
     }
     }
-
     // Find the timestamp of the closest vsync. This is the vsync that we're targeting.
     // Find the timestamp of the closest vsync. This is the vsync that we're targeting.
     long snappedTimeNs = closestVsync(adjustedReleaseTimeNs, sampledVsyncTimeNs, vsyncDurationNs);
     long snappedTimeNs = closestVsync(adjustedReleaseTimeNs, sampledVsyncTimeNs, vsyncDurationNs);
     // Apply an offset so that we release before the target vsync, but after the previous one.
     // Apply an offset so that we release before the target vsync, but after the previous one.
     return snappedTimeNs - vsyncOffsetNs;
     return snappedTimeNs - vsyncOffsetNs;
   }
   }
 
 
+  private void resetAdjustment() {
+    frameIndex = 0;
+    lastAdjustedFrameIndex = C.INDEX_UNSET;
+    pendingLastAdjustedFrameIndex = C.INDEX_UNSET;
+  }
+
   @RequiresApi(17)
   @RequiresApi(17)
   private DefaultDisplayListener maybeBuildDefaultDisplayListenerV17(Context context) {
   private DefaultDisplayListener maybeBuildDefaultDisplayListenerV17(Context context) {
     DisplayManager manager = (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE);
     DisplayManager manager = (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE);