Add the possility to shift frame timestamps in SampleConsumer

This is needed for constrained multi-asset to shift the timestamps of
the media items that are not the first in the sequence.

PiperOrigin-RevId: 502409923

library/common/src/main/java/com/google/android/exoplayer2/util/FrameInfo.java

@@ -17,8 +17,95 @@ package com.google.android.exoplayer2.util;
 
 
 import static com.google.android.exoplayer2.util.Assertions.checkArgument;
 import static com.google.android.exoplayer2.util.Assertions.checkArgument;
 
 
+import com.google.errorprone.annotations.CanIgnoreReturnValue;
+
 /** Value class specifying information about a decoded video frame. */
 /** Value class specifying information about a decoded video frame. */
 public class FrameInfo {
 public class FrameInfo {
+
+  /** A builder for {@link FrameInfo} instances. */
+  public static final class Builder {
+
+    private int width;
+    private int height;
+    private float pixelWidthHeightRatio;
+    private long streamOffsetUs;
+    private long offsetToAddUs;
+
+    /**
+     * Creates an instance with default values.
+     *
+     * @param width The frame width, in pixels.
+     * @param height The frame height, in pixels.
+     */
+    public Builder(int width, int height) {
+      this.width = width;
+      this.height = height;
+      pixelWidthHeightRatio = 1;
+    }
+
+    /** Creates an instance with the values of the provided {@link FrameInfo}. */
+    public Builder(FrameInfo frameInfo) {
+      width = frameInfo.width;
+      height = frameInfo.height;
+      pixelWidthHeightRatio = frameInfo.pixelWidthHeightRatio;
+      streamOffsetUs = frameInfo.streamOffsetUs;
+      offsetToAddUs = frameInfo.offsetToAddUs;
+    }
+
+    /** Sets the frame width, in pixels. */
+    @CanIgnoreReturnValue
+    public Builder setWidth(int width) {
+      this.width = width;
+      return this;
+    }
+
+    /** Sets the frame height, in pixels. */
+    @CanIgnoreReturnValue
+    public Builder setHeight(int height) {
+      this.height = height;
+      return this;
+    }
+
+    /**
+     * Sets the ratio of width over height for each pixel.
+     *
+     * <p>The default value is {@code 1}.
+     */
+    @CanIgnoreReturnValue
+    public Builder setPixelWidthHeightRatio(float pixelWidthHeightRatio) {
+      this.pixelWidthHeightRatio = pixelWidthHeightRatio;
+      return this;
+    }
+
+    /**
+     * Sets the {@linkplain FrameInfo#streamOffsetUs stream offset}, in microseconds.
+     *
+     * <p>The default value is {@code 0}.
+     */
+    @CanIgnoreReturnValue
+    public Builder setStreamOffsetUs(long streamOffsetUs) {
+      this.streamOffsetUs = streamOffsetUs;
+      return this;
+    }
+
+    /**
+     * Sets the {@linkplain FrameInfo#offsetToAddUs offset to add} to the frame presentation
+     * timestamp, in microseconds.
+     *
+     * <p>The default value is {@code 0}.
+     */
+    @CanIgnoreReturnValue
+    public Builder setOffsetToAddUs(long offsetToAddUs) {
+      this.offsetToAddUs = offsetToAddUs;
+      return this;
+    }
+
+    /** Builds a {@link FrameInfo} instance. */
+    public FrameInfo build() {
+      return new FrameInfo(width, height, pixelWidthHeightRatio, streamOffsetUs, offsetToAddUs);
+    }
+  }
+
   /** The width of the frame, in pixels. */
   /** The width of the frame, in pixels. */
   public final int width;
   public final int width;
   /** The height of the frame, in pixels. */
   /** The height of the frame, in pixels. */
@@ -29,23 +116,24 @@ public class FrameInfo {
    * An offset in microseconds that is part of the input timestamps and should be ignored for
    * An offset in microseconds that is part of the input timestamps and should be ignored for
    * processing but added back to the output timestamps.
    * processing but added back to the output timestamps.
    *
    *
-   * <p>The offset stays constant within a stream but changes in between streams to ensure that
-   * frame timestamps are always monotonically increasing.
+   * <p>The offset stays constant within a stream. If the first timestamp of the next stream is less
+   * than or equal to the last timestamp of the current stream (including the {@linkplain
+   * #offsetToAddUs} offset to add), the stream offset must be updated between the streams to ensure
+   * that the offset frame timestamps are always monotonically increasing.
    */
    */
   public final long streamOffsetUs;
   public final long streamOffsetUs;
-
-  // TODO(b/227624622): Add color space information for HDR.
-
   /**
   /**
-   * Creates a new instance.
+   * The offset that must be added to the frame presentation timestamp, in microseconds.
    *
    *
-   * @param width The width of the frame, in pixels.
-   * @param height The height of the frame, in pixels.
-   * @param pixelWidthHeightRatio The ratio of width over height for each pixel.
-   * @param streamOffsetUs An offset in microseconds that is part of the input timestamps and should
-   *     be ignored for processing but added back to the output timestamps.
+   * <p>This offset is not part of the input timestamps. It is added to the frame timestamps before
+   * processing, and is retained in the output timestamps.
    */
    */
-  public FrameInfo(int width, int height, float pixelWidthHeightRatio, long streamOffsetUs) {
+  public final long offsetToAddUs;
+
+  // TODO(b/227624622): Add color space information for HDR.
+
+  private FrameInfo(
+      int width, int height, float pixelWidthHeightRatio, long streamOffsetUs, long offsetToAddUs) {
     checkArgument(width > 0, "width must be positive, but is: " + width);
     checkArgument(width > 0, "width must be positive, but is: " + width);
     checkArgument(height > 0, "height must be positive, but is: " + height);
     checkArgument(height > 0, "height must be positive, but is: " + height);
 
 
@@ -53,5 +141,6 @@ public class FrameInfo {
     this.height = height;
     this.height = height;
     this.pixelWidthHeightRatio = pixelWidthHeightRatio;
     this.pixelWidthHeightRatio = pixelWidthHeightRatio;
     this.streamOffsetUs = streamOffsetUs;
     this.streamOffsetUs = streamOffsetUs;
+    this.offsetToAddUs = offsetToAddUs;
   }
   }
 }
 }

library/common/src/main/java/com/google/android/exoplayer2/util/FrameProcessor.java

@@ -135,8 +135,9 @@ public interface FrameProcessor {
    * <p>Pixels are expanded using the {@link FrameInfo#pixelWidthHeightRatio} so that the output
    * <p>Pixels are expanded using the {@link FrameInfo#pixelWidthHeightRatio} so that the output
    * frames' pixels have a ratio of 1.
    * frames' pixels have a ratio of 1.
    *
    *
-   * <p>The caller should update {@link FrameInfo#streamOffsetUs} when switching input streams to
-   * ensure that frame timestamps are always monotonically increasing.
+   * <p>The caller should update {@link FrameInfo#streamOffsetUs} when switching to an input stream
+   * whose first frame timestamp is less than or equal to the last timestamp received. This stream
+   * offset should ensure that frame timestamps are monotonically increasing.
    *
    *
    * <p>Can be called on any thread.
    * <p>Can be called on any thread.
    */
    */

library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java

@@ -2060,11 +2060,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
     public void setInputFormat(Format inputFormat) {
     public void setInputFormat(Format inputFormat) {
       checkNotNull(frameProcessor)
       checkNotNull(frameProcessor)
           .setInputFrameInfo(
           .setInputFrameInfo(
-              new FrameInfo(
-                  inputFormat.width,
-                  inputFormat.height,
-                  inputFormat.pixelWidthHeightRatio,
-                  renderer.getOutputStreamOffsetUs()));
+              new FrameInfo.Builder(inputFormat.width, inputFormat.height)
+                  .setPixelWidthHeightRatio(inputFormat.pixelWidthHeightRatio)
+                  .setStreamOffsetUs(renderer.getOutputStreamOffsetUs())
+                  .build());
       this.inputFormat = inputFormat;
       this.inputFormat = inputFormat;
 
 
       if (registeredLastFrame) {
       if (registeredLastFrame) {

library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessorFrameReleaseTest.java

@@ -341,9 +341,7 @@ public final class GlEffectsFrameProcessorFrameReleaseTest {
             () -> {
             () -> {
               blankFrameProducer.configureGlObjects();
               blankFrameProducer.configureGlObjects();
               checkNotNull(glEffectsFrameProcessor)
               checkNotNull(glEffectsFrameProcessor)
-                  .setInputFrameInfo(
-                      new FrameInfo(
-                          WIDTH, HEIGHT, /* pixelWidthHeightRatio= */ 1, /* streamOffsetUs= */ 0));
+                  .setInputFrameInfo(new FrameInfo.Builder(WIDTH, HEIGHT).build());
               // A frame needs to be registered despite not queuing any external input to ensure
               // A frame needs to be registered despite not queuing any external input to ensure
               // that
               // that
               // the frame processor knows about the stream offset.
               // the frame processor knows about the stream offset.

library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessorPixelTest.java

@@ -739,11 +739,11 @@ public final class GlEffectsFrameProcessorPixelTest {
             @Override
             @Override
             public void onContainerExtracted(MediaFormat mediaFormat) {
             public void onContainerExtracted(MediaFormat mediaFormat) {
               glEffectsFrameProcessor.setInputFrameInfo(
               glEffectsFrameProcessor.setInputFrameInfo(
-                  new FrameInfo(
+                  new FrameInfo.Builder(
                           mediaFormat.getInteger(MediaFormat.KEY_WIDTH),
                           mediaFormat.getInteger(MediaFormat.KEY_WIDTH),
-                      mediaFormat.getInteger(MediaFormat.KEY_HEIGHT),
-                      pixelWidthHeightRatio,
-                      /* streamOffsetUs= */ 0));
+                          mediaFormat.getInteger(MediaFormat.KEY_HEIGHT))
+                      .setPixelWidthHeightRatio(pixelWidthHeightRatio)
+                      .build());
               glEffectsFrameProcessor.registerInputFrame();
               glEffectsFrameProcessor.registerInputFrame();
             }
             }
 
 

library/effect/src/main/java/com/google/android/exoplayer2/effect/ExternalTextureManager.java

@@ -160,6 +160,7 @@ import java.util.concurrent.atomic.AtomicInteger;
     surfaceTexture.getTransformMatrix(textureTransformMatrix);
     surfaceTexture.getTransformMatrix(textureTransformMatrix);
     externalTextureProcessor.setTextureTransformMatrix(textureTransformMatrix);
     externalTextureProcessor.setTextureTransformMatrix(textureTransformMatrix);
     long frameTimeNs = surfaceTexture.getTimestamp();
     long frameTimeNs = surfaceTexture.getTimestamp();
+    long offsetToAddUs = currentFrame.offsetToAddUs;
     long streamOffsetUs = currentFrame.streamOffsetUs;
     long streamOffsetUs = currentFrame.streamOffsetUs;
     if (streamOffsetUs != previousStreamOffsetUs) {
     if (streamOffsetUs != previousStreamOffsetUs) {
       if (previousStreamOffsetUs != C.TIME_UNSET) {
       if (previousStreamOffsetUs != C.TIME_UNSET) {
@@ -167,8 +168,8 @@ import java.util.concurrent.atomic.AtomicInteger;
       }
       }
       previousStreamOffsetUs = streamOffsetUs;
       previousStreamOffsetUs = streamOffsetUs;
     }
     }
-    // Correct for the stream offset so processors see original media presentation timestamps.
-    long presentationTimeUs = (frameTimeNs / 1000) - streamOffsetUs;
+    // Correct the presentation time so that processors don't see the stream offset.
+    long presentationTimeUs = (frameTimeNs / 1000) + offsetToAddUs - streamOffsetUs;
     externalTextureProcessor.queueInputFrame(
     externalTextureProcessor.queueInputFrame(
         new TextureInfo(
         new TextureInfo(
             externalTexId, /* fboId= */ C.INDEX_UNSET, currentFrame.width, currentFrame.height),
             externalTexId, /* fboId= */ C.INDEX_UNSET, currentFrame.width, currentFrame.height),

library/effect/src/main/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessor.java

@@ -438,23 +438,21 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
   }
   }
 
 
   /**
   /**
-   * Expands or shrinks the frame based on the {@link FrameInfo#pixelWidthHeightRatio} and returns a
-   * new {@link FrameInfo} instance with scaled dimensions and {@link
-   * FrameInfo#pixelWidthHeightRatio} of {@code 1}.
+   * Expands the frame based on the {@link FrameInfo#pixelWidthHeightRatio} and returns a new {@link
+   * FrameInfo} instance with scaled dimensions and {@link FrameInfo#pixelWidthHeightRatio} of
+   * {@code 1}.
    */
    */
   private FrameInfo adjustForPixelWidthHeightRatio(FrameInfo frameInfo) {
   private FrameInfo adjustForPixelWidthHeightRatio(FrameInfo frameInfo) {
     if (frameInfo.pixelWidthHeightRatio > 1f) {
     if (frameInfo.pixelWidthHeightRatio > 1f) {
-      return new FrameInfo(
-          (int) (frameInfo.width * frameInfo.pixelWidthHeightRatio),
-          frameInfo.height,
-          /* pixelWidthHeightRatio= */ 1,
-          frameInfo.streamOffsetUs);
+      return new FrameInfo.Builder(frameInfo)
+          .setWidth((int) (frameInfo.width * frameInfo.pixelWidthHeightRatio))
+          .setPixelWidthHeightRatio(1)
+          .build();
     } else if (frameInfo.pixelWidthHeightRatio < 1f) {
     } else if (frameInfo.pixelWidthHeightRatio < 1f) {
-      return new FrameInfo(
-          frameInfo.width,
-          (int) (frameInfo.height / frameInfo.pixelWidthHeightRatio),
-          /* pixelWidthHeightRatio= */ 1,
-          frameInfo.streamOffsetUs);
+      return new FrameInfo.Builder(frameInfo)
+          .setHeight((int) (frameInfo.height / frameInfo.pixelWidthHeightRatio))
+          .setPixelWidthHeightRatio(1)
+          .build();
     } else {
     } else {
       return frameInfo;
       return frameInfo;
     }
     }

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java

@@ -222,8 +222,10 @@ import org.checkerframework.dataflow.qual.Pure;
           e, TransformationException.ERROR_CODE_FRAME_PROCESSING_FAILED);
           e, TransformationException.ERROR_CODE_FRAME_PROCESSING_FAILED);
     }
     }
     frameProcessor.setInputFrameInfo(
     frameProcessor.setInputFrameInfo(
-        new FrameInfo(
-            decodedWidth, decodedHeight, inputFormat.pixelWidthHeightRatio, streamOffsetUs));
+        new FrameInfo.Builder(decodedWidth, decodedHeight)
+            .setPixelWidthHeightRatio(inputFormat.pixelWidthHeightRatio)
+            .setStreamOffsetUs(streamOffsetUs)
+            .build());
   }
   }
 
 
   @Override
   @Override