Skip to content

SDK / exoplayer

Go to a project
Commit d838f2c6 by claincly Committed by Ian Baker
Options

Allow setting individual offset for bitmaps. 

PiperOrigin-RevId: 527001582
parent c1e583cb
Inline Side-by-side
Showing with 116 additions and 14 deletions
library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java
...@@ -66,11 +66,20 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest { ...@@ -66,11 +66,20 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest {
videoFrameProcessorTestRunner = getDefaultFrameProcessorTestRunnerBuilder(testId).build(); videoFrameProcessorTestRunner = getDefaultFrameProcessorTestRunnerBuilder(testId).build();
videoFrameProcessorTestRunner.queueInputBitmap( videoFrameProcessorTestRunner.queueInputBitmap(
readBitmap(ORIGINAL_PNG_ASSET_PATH), C.MICROS_PER_SECOND, /* frameRate= */ 2); readBitmap(ORIGINAL_PNG_ASSET_PATH),
C.MICROS_PER_SECOND,
/* offsetToAddUs= */ 0L,
/* frameRate= */ 2);
videoFrameProcessorTestRunner.queueInputBitmap( videoFrameProcessorTestRunner.queueInputBitmap(
readBitmap(SCALE_WIDE_PNG_ASSET_PATH), 2 * C.MICROS_PER_SECOND, /* frameRate= */ 3); readBitmap(SCALE_WIDE_PNG_ASSET_PATH),
2 * C.MICROS_PER_SECOND,
/* offsetToAddUs= */ 0L,
/* frameRate= */ 3);
videoFrameProcessorTestRunner.queueInputBitmap( videoFrameProcessorTestRunner.queueInputBitmap(
readBitmap(BITMAP_OVERLAY_PNG_ASSET_PATH), 3 * C.MICROS_PER_SECOND, /* frameRate= */ 4); readBitmap(BITMAP_OVERLAY_PNG_ASSET_PATH),
3 * C.MICROS_PER_SECOND,
/* offsetToAddUs= */ 0L,
/* frameRate= */ 4);
videoFrameProcessorTestRunner.endFrameProcessingAndGetImage(); videoFrameProcessorTestRunner.endFrameProcessingAndGetImage();
int actualFrameCount = framesProduced.get(); int actualFrameCount = framesProduced.get();
...@@ -87,6 +96,7 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest { ...@@ -87,6 +96,7 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest {
videoFrameProcessorTestRunner.queueInputBitmap( videoFrameProcessorTestRunner.queueInputBitmap(
readBitmap(ORIGINAL_PNG_ASSET_PATH), readBitmap(ORIGINAL_PNG_ASSET_PATH),
/* durationUs= */ C.MICROS_PER_SECOND, /* durationUs= */ C.MICROS_PER_SECOND,
/* offsetToAddUs= */ 0L,
/* frameRate= */ 1); /* frameRate= */ 1);
} }
videoFrameProcessorTestRunner.endFrameProcessingAndGetImage(); videoFrameProcessorTestRunner.endFrameProcessingAndGetImage();
...@@ -97,6 +107,63 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest { ...@@ -97,6 +107,63 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest {
@RequiresNonNull("framesProduced") @RequiresNonNull("framesProduced")
@Test @Test
public void imageInput_queueOneWithStartOffset_outputsFramesAtTheCorrectPresentationTimesUs()
throws Exception {
String testId =
"imageInput_queueOneWithStartOffset_outputsFramesAtTheCorrectPresentationTimesUs";
Queue<Long> actualPresentationTimesUs = new ConcurrentLinkedQueue<>();
videoFrameProcessorTestRunner =
getDefaultFrameProcessorTestRunnerBuilder(testId)
.setOnOutputFrameAvailableListener(actualPresentationTimesUs::add)
.build();
long offsetUs = 1_000_000L;
videoFrameProcessorTestRunner.queueInputBitmap(
readBitmap(ORIGINAL_PNG_ASSET_PATH),
/* durationUs= */ C.MICROS_PER_SECOND,
/* offsetToAddUs= */ offsetUs,
/* frameRate= */ 2);
videoFrameProcessorTestRunner.endFrameProcessingAndGetImage();
assertThat(actualPresentationTimesUs)
.containsExactly(offsetUs, offsetUs + C.MICROS_PER_SECOND / 2)
.inOrder();
}
@RequiresNonNull("framesProduced")
@Test
public void imageInput_queueWithStartOffsets_outputsFramesAtTheCorrectPresentationTimesUs()
throws Exception {
String testId = "imageInput_queueWithStartOffsets_outputsFramesAtTheCorrectPresentationTimesUs";
Queue<Long> actualPresentationTimesUs = new ConcurrentLinkedQueue<>();
videoFrameProcessorTestRunner =
getDefaultFrameProcessorTestRunnerBuilder(testId)
.setOnOutputFrameAvailableListener(actualPresentationTimesUs::add)
.build();
long offsetUs1 = 1_000_000L;
videoFrameProcessorTestRunner.queueInputBitmap(
readBitmap(ORIGINAL_PNG_ASSET_PATH),
/* durationUs= */ C.MICROS_PER_SECOND,
/* offsetToAddUs= */ offsetUs1,
/* frameRate= */ 2);
long offsetUs2 = 2_000_000L;
videoFrameProcessorTestRunner.queueInputBitmap(
readBitmap(SCALE_WIDE_PNG_ASSET_PATH),
/* durationUs= */ C.MICROS_PER_SECOND,
/* offsetToAddUs= */ offsetUs2,
/* frameRate= */ 2);
videoFrameProcessorTestRunner.endFrameProcessingAndGetImage();
assertThat(actualPresentationTimesUs)
.containsExactly(
offsetUs1,
offsetUs1 + C.MICROS_PER_SECOND / 2,
offsetUs2,
offsetUs2 + C.MICROS_PER_SECOND / 2)
.inOrder();
}
@RequiresNonNull("framesProduced")
@Test
public void public void
imageInput_queueEndAndQueueAgain_outputsFirstSetOfFramesOnlyAtTheCorrectPresentationTimesUs() imageInput_queueEndAndQueueAgain_outputsFirstSetOfFramesOnlyAtTheCorrectPresentationTimesUs()
throws Exception { throws Exception {
...@@ -111,11 +178,13 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest { ...@@ -111,11 +178,13 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest {
videoFrameProcessorTestRunner.queueInputBitmap( videoFrameProcessorTestRunner.queueInputBitmap(
readBitmap(ORIGINAL_PNG_ASSET_PATH), readBitmap(ORIGINAL_PNG_ASSET_PATH),
/* durationUs= */ C.MICROS_PER_SECOND, /* durationUs= */ C.MICROS_PER_SECOND,
/* offsetToAddUs= */ 0L,
/* frameRate= */ 2); /* frameRate= */ 2);
videoFrameProcessorTestRunner.endFrameProcessingAndGetImage(); videoFrameProcessorTestRunner.endFrameProcessingAndGetImage();
videoFrameProcessorTestRunner.queueInputBitmap( videoFrameProcessorTestRunner.queueInputBitmap(
readBitmap(ORIGINAL_PNG_ASSET_PATH), readBitmap(ORIGINAL_PNG_ASSET_PATH),
/* durationUs= */ 2 * C.MICROS_PER_SECOND, /* durationUs= */ 2 * C.MICROS_PER_SECOND,
/* offsetToAddUs= */ 0L,
/* frameRate= */ 3); /* frameRate= */ 3);
assertThat(actualPresentationTimesUs).containsExactly(0L, C.MICROS_PER_SECOND / 2).inOrder(); assertThat(actualPresentationTimesUs).containsExactly(0L, C.MICROS_PER_SECOND / 2).inOrder();
......
library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessorPixelTest.java
...@@ -120,7 +120,7 @@ public final class DefaultVideoFrameProcessorPixelTest { ...@@ -120,7 +120,7 @@ public final class DefaultVideoFrameProcessorPixelTest {
Bitmap expectedBitmap = readBitmap(IMAGE_TO_VIDEO_PNG_ASSET_PATH); Bitmap expectedBitmap = readBitmap(IMAGE_TO_VIDEO_PNG_ASSET_PATH);
videoFrameProcessorTestRunner.queueInputBitmap( videoFrameProcessorTestRunner.queueInputBitmap(
originalBitmap, C.MICROS_PER_SECOND, /* frameRate= */ 1); originalBitmap, C.MICROS_PER_SECOND, /* offsetToAddUs= */ 0L, /* frameRate= */ 1);
Bitmap actualBitmap = videoFrameProcessorTestRunner.endFrameProcessingAndGetImage(); Bitmap actualBitmap = videoFrameProcessorTestRunner.endFrameProcessingAndGetImage();
// TODO(b/207848601): Switch to using proper tooling for testing against golden data. // TODO(b/207848601): Switch to using proper tooling for testing against golden data.
...@@ -148,7 +148,7 @@ public final class DefaultVideoFrameProcessorPixelTest { ...@@ -148,7 +148,7 @@ public final class DefaultVideoFrameProcessorPixelTest {
Bitmap expectedBitmap = readBitmap(IMAGE_TO_CROPPED_VIDEO_PNG_ASSET_PATH); Bitmap expectedBitmap = readBitmap(IMAGE_TO_CROPPED_VIDEO_PNG_ASSET_PATH);
videoFrameProcessorTestRunner.queueInputBitmap( videoFrameProcessorTestRunner.queueInputBitmap(
originalBitmap, C.MICROS_PER_SECOND, /* frameRate= */ 1); originalBitmap, C.MICROS_PER_SECOND, /* offsetToAddUs= */ 0L, /* frameRate= */ 1);
Bitmap actualBitmap = videoFrameProcessorTestRunner.endFrameProcessingAndGetImage(); Bitmap actualBitmap = videoFrameProcessorTestRunner.endFrameProcessingAndGetImage();
// TODO(b/207848601): Switch to using proper tooling for testing against golden data. // TODO(b/207848601): Switch to using proper tooling for testing against golden data.
......
library/effect/src/main/java/com/google/android/exoplayer2/effect/BitmapTextureManager.java
...@@ -77,9 +77,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -77,9 +77,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@Override @Override
public void queueInputBitmap( public void queueInputBitmap(
Bitmap inputBitmap, long durationUs, float frameRate, boolean useHdr) { Bitmap inputBitmap, long durationUs, long offsetUs, float frameRate, boolean useHdr) {
videoFrameProcessingTaskExecutor.submit( videoFrameProcessingTaskExecutor.submit(
() -> setupBitmap(inputBitmap, durationUs, frameRate, useHdr)); () -> setupBitmap(inputBitmap, durationUs, offsetUs, frameRate, useHdr));
} }
@Override @Override
...@@ -114,7 +114,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -114,7 +114,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
// Methods that must be called on the GL thread. // Methods that must be called on the GL thread.
private void setupBitmap(Bitmap bitmap, long durationUs, float frameRate, boolean useHdr) private void setupBitmap(
Bitmap bitmap, long durationUs, long offsetUs, float frameRate, boolean useHdr)
throws VideoFrameProcessingException { throws VideoFrameProcessingException {
this.useHdr = useHdr; this.useHdr = useHdr;
if (inputEnded) { if (inputEnded) {
...@@ -122,7 +123,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -122,7 +123,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
int framesToAdd = round(frameRate * (durationUs / (float) C.MICROS_PER_SECOND)); int framesToAdd = round(frameRate * (durationUs / (float) C.MICROS_PER_SECOND));
double frameDurationUs = C.MICROS_PER_SECOND / frameRate; double frameDurationUs = C.MICROS_PER_SECOND / frameRate;
pendingBitmaps.add(new BitmapFrameSequenceInfo(bitmap, frameDurationUs, framesToAdd)); pendingBitmaps.add(new BitmapFrameSequenceInfo(bitmap, offsetUs, frameDurationUs, framesToAdd));
maybeQueueToShaderProgram(); maybeQueueToShaderProgram();
} }
...@@ -136,6 +137,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -136,6 +137,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
if (framesToQueueForCurrentBitmap == 0) { if (framesToQueueForCurrentBitmap == 0) {
Bitmap bitmap = currentBitmapInfo.bitmap; Bitmap bitmap = currentBitmapInfo.bitmap;
framesToQueueForCurrentBitmap = currentBitmapInfo.numberOfFrames; framesToQueueForCurrentBitmap = currentBitmapInfo.numberOfFrames;
currentPresentationTimeUs = currentBitmapInfo.offsetUs;
int currentTexId; int currentTexId;
try { try {
if (currentGlTextureInfo != null) { if (currentGlTextureInfo != null) {
...@@ -187,11 +189,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -187,11 +189,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** Information to generate all the frames associated with a specific {@link Bitmap}. */ /** Information to generate all the frames associated with a specific {@link Bitmap}. */
private static final class BitmapFrameSequenceInfo { private static final class BitmapFrameSequenceInfo {
public final Bitmap bitmap; public final Bitmap bitmap;
public final long offsetUs;
public final double frameDurationUs; public final double frameDurationUs;
public final int numberOfFrames; public final int numberOfFrames;
public BitmapFrameSequenceInfo(Bitmap bitmap, double frameDurationUs, int numberOfFrames) { public BitmapFrameSequenceInfo(
Bitmap bitmap, long offsetUs, double frameDurationUs, int numberOfFrames) {
this.bitmap = bitmap; this.bitmap = bitmap;
this.offsetUs = offsetUs;
this.frameDurationUs = frameDurationUs; this.frameDurationUs = frameDurationUs;
this.numberOfFrames = numberOfFrames; this.numberOfFrames = numberOfFrames;
} }
......
library/effect/src/main/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessor.java
...@@ -17,6 +17,7 @@ package com.google.android.exoplayer2.effect; ...@@ -17,6 +17,7 @@ package com.google.android.exoplayer2.effect;
import static androidx.annotation.VisibleForTesting.PACKAGE_PRIVATE; import static androidx.annotation.VisibleForTesting.PACKAGE_PRIVATE;
import static com.google.android.exoplayer2.util.Assertions.checkArgument; import static com.google.android.exoplayer2.util.Assertions.checkArgument;
import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import static com.google.android.exoplayer2.util.Assertions.checkState; import static com.google.android.exoplayer2.util.Assertions.checkState;
import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
import static com.google.common.collect.Iterables.getLast; import static com.google.common.collect.Iterables.getLast;
...@@ -251,6 +252,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -251,6 +252,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
private volatile @MonotonicNonNull FrameInfo nextInputFrameInfo; private volatile @MonotonicNonNull FrameInfo nextInputFrameInfo;
private volatile boolean inputStreamEnded; private volatile boolean inputStreamEnded;
private volatile boolean hasRefreshedNextInputFrameInfo;
private DefaultVideoFrameProcessor( private DefaultVideoFrameProcessor(
EGLDisplay eglDisplay, EGLDisplay eglDisplay,
...@@ -319,7 +321,16 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -319,7 +321,16 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
@Override @Override
public void queueInputBitmap(Bitmap inputBitmap, long durationUs, float frameRate) { public void queueInputBitmap(Bitmap inputBitmap, long durationUs, float frameRate) {
inputHandler.queueInputBitmap(inputBitmap, durationUs, frameRate, /* useHdr= */ false); checkState(
hasRefreshedNextInputFrameInfo,
"setInputFrameInfo must be called before queueing another bitmap");
inputHandler.queueInputBitmap(
inputBitmap,
durationUs,
checkNotNull(nextInputFrameInfo).offsetToAddUs,
frameRate,
/* useHdr= */ false);
hasRefreshedNextInputFrameInfo = false;
} }
@Override @Override
...@@ -330,6 +341,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -330,6 +341,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
@Override @Override
public void setInputFrameInfo(FrameInfo inputFrameInfo) { public void setInputFrameInfo(FrameInfo inputFrameInfo) {
nextInputFrameInfo = adjustForPixelWidthHeightRatio(inputFrameInfo); nextInputFrameInfo = adjustForPixelWidthHeightRatio(inputFrameInfo);
hasRefreshedNextInputFrameInfo = true;
} }
@Override @Override
...@@ -339,6 +351,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ...@@ -339,6 +351,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
nextInputFrameInfo, "setInputFrameInfo must be called before registering input frames"); nextInputFrameInfo, "setInputFrameInfo must be called before registering input frames");
inputHandler.registerInputFrame(nextInputFrameInfo); inputHandler.registerInputFrame(nextInputFrameInfo);
hasRefreshedNextInputFrameInfo = false;
} }
@Override @Override
......
library/effect/src/main/java/com/google/android/exoplayer2/effect/ExternalTextureManager.java
...@@ -17,6 +17,7 @@ package com.google.android.exoplayer2.effect; ...@@ -17,6 +17,7 @@ package com.google.android.exoplayer2.effect;
import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
import android.graphics.Bitmap;
import android.graphics.SurfaceTexture; import android.graphics.SurfaceTexture;
import android.view.Surface; import android.view.Surface;
import androidx.annotation.Nullable; import androidx.annotation.Nullable;
...@@ -110,6 +111,12 @@ import java.util.concurrent.atomic.AtomicInteger; ...@@ -110,6 +111,12 @@ import java.util.concurrent.atomic.AtomicInteger;
} }
@Override @Override
public void queueInputBitmap(
Bitmap inputBitmap, long durationUs, long offsetUs, float frameRate, boolean useHdr) {
throw new UnsupportedOperationException();
}
@Override
public Surface getInputSurface() { public Surface getInputSurface() {
return surface; return surface;
} }
......
library/effect/src/main/java/com/google/android/exoplayer2/effect/InputHandler.java
...@@ -39,10 +39,16 @@ import com.google.android.exoplayer2.util.VideoFrameProcessor; ...@@ -39,10 +39,16 @@ import com.google.android.exoplayer2.util.VideoFrameProcessor;
/** /**
* Provides an input {@link Bitmap} to put into the video frames. * Provides an input {@link Bitmap} to put into the video frames.
* *
* @see VideoFrameProcessor#queueInputBitmap * @param inputBitmap The {@link Bitmap} queued to the {@code VideoFrameProcessor}.
* @param durationUs The duration for which to display the {@code inputBitmap}, in microseconds.
* @param offsetUs The offset, from the start of the input stream, to apply for the {@code
* inputBitmap} in microseconds.
* @param frameRate The frame rate at which to display the {@code inputBitmap}, in frames per
* second.
* @param useHdr Whether input and/or output colors are HDR.
*/ */
default void queueInputBitmap( default void queueInputBitmap(
Bitmap inputBitmap, long durationUs, float frameRate, boolean useHdr) { Bitmap inputBitmap, long durationUs, long offsetUs, float frameRate, boolean useHdr) {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
......
testutils/src/main/java/com/google/android/exoplayer2/testutil/VideoFrameProcessorTestRunner.java
...@@ -331,10 +331,12 @@ public final class VideoFrameProcessorTestRunner { ...@@ -331,10 +331,12 @@ public final class VideoFrameProcessorTestRunner {
return endFrameProcessingAndGetImage(); return endFrameProcessingAndGetImage();
} }
public void queueInputBitmap(Bitmap inputBitmap, long durationUs, float frameRate) { public void queueInputBitmap(
Bitmap inputBitmap, long durationUs, long offsetToAddUs, float frameRate) {
videoFrameProcessor.setInputFrameInfo( videoFrameProcessor.setInputFrameInfo(
new FrameInfo.Builder(inputBitmap.getWidth(), inputBitmap.getHeight()) new FrameInfo.Builder(inputBitmap.getWidth(), inputBitmap.getHeight())
.setPixelWidthHeightRatio(pixelWidthHeightRatio) .setPixelWidthHeightRatio(pixelWidthHeightRatio)
.setOffsetToAddUs(offsetToAddUs)
.build()); .build());
videoFrameProcessor.queueInputBitmap(inputBitmap, durationUs, frameRate); videoFrameProcessor.queueInputBitmap(inputBitmap, durationUs, frameRate);
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment