Skip to content

SDK / exoplayer

Go to a project
Commit 8aa1638f by tofunmi Committed by Andrew Lewis
Options

Add multi-image input FrameProcessor tests 

PiperOrigin-RevId: 510441777
parent bd4a5ecf
Inline Side-by-side
Showing with 164 additions and 20 deletions
library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java 0 → 100644
/*
* Copyright 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.effect;
import static com.google.android.exoplayer2.testutil.BitmapPixelTestUtil.readBitmap;
import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import static com.google.common.truth.Truth.assertThat;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.testutil.VideoFrameProcessorTestRunner;
import java.util.concurrent.atomic.AtomicInteger;
import org.checkerframework.checker.nullness.qual.EnsuresNonNull;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
/** Tests for frame queuing and output in {@link DefaultVideoFrameProcessor} given image input. */
@RunWith(AndroidJUnit4.class)
public class DefaultVideoFrameProcessorImageFrameOutputTest {
public static final String ORIGINAL_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/electrical_colors/original.png";
public static final String WRAPPED_CROP_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/electrical_colors/image_input_with_wrapped_crop.png";
public static final String BITMAP_OVERLAY_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_FrameProcessor.png";
private @MonotonicNonNull VideoFrameProcessorTestRunner videoFrameProcessorTestRunner;
private @MonotonicNonNull AtomicInteger framesProduced;
@EnsuresNonNull("framesProduced")
@Before
public void setUp() {
framesProduced = new AtomicInteger();
}
@After
public void release() {
checkNotNull(videoFrameProcessorTestRunner).release();
}
@RequiresNonNull("framesProduced")
@Test
public void imageInput_queueThreeBitmaps_outputsAllFrames() throws Exception {
String testId = "imageInput_withThreeBitmaps_outputsAllFrames";
videoFrameProcessorTestRunner = getDefaultFrameProcessorTestRunnerBuilder(testId).build();
videoFrameProcessorTestRunner.queueInputBitmap(
readBitmap(ORIGINAL_PNG_ASSET_PATH), C.MICROS_PER_SECOND, /* frameRate= */ 2);
videoFrameProcessorTestRunner.queueInputBitmap(
readBitmap(WRAPPED_CROP_PNG_ASSET_PATH), 2 * C.MICROS_PER_SECOND, /* frameRate= */ 3);
videoFrameProcessorTestRunner.queueInputBitmap(
readBitmap(BITMAP_OVERLAY_PNG_ASSET_PATH), 3 * C.MICROS_PER_SECOND, /* frameRate= */ 4);
videoFrameProcessorTestRunner.endFrameProcessingAndGetImage();
int actualFrameCount = framesProduced.get();
assertThat(actualFrameCount).isEqualTo(/* expected= */ 20);
}
@RequiresNonNull("framesProduced")
@Test
public void imageInput_queueTwentyBitmaps_outputsAllFrames() throws Exception {
String testId = "imageInput_queueTwentyBitmaps_outputsAllFrames";
videoFrameProcessorTestRunner = getDefaultFrameProcessorTestRunnerBuilder(testId).build();
for (int i = 0; i < 20; i++) {
videoFrameProcessorTestRunner.queueInputBitmap(
readBitmap(ORIGINAL_PNG_ASSET_PATH),
/* durationUs= */ C.MICROS_PER_SECOND,
/* frameRate= */ 1);
}
videoFrameProcessorTestRunner.endFrameProcessingAndGetImage();
int actualFrameCount = framesProduced.get();
assertThat(actualFrameCount).isEqualTo(/* expected= */ 20);
}
@RequiresNonNull("framesProduced")
@Test
public void imageInput_queueEndAndQueueAgain_outputsFirstSetOfFramesOnly() throws Exception {
String testId = "imageInput_queueEndAndQueueAgain_outputsFirstSetOfFramesOnly";
videoFrameProcessorTestRunner = getDefaultFrameProcessorTestRunnerBuilder(testId).build();
videoFrameProcessorTestRunner.queueInputBitmap(
readBitmap(ORIGINAL_PNG_ASSET_PATH),
/* durationUs= */ C.MICROS_PER_SECOND,
/* frameRate= */ 2);
videoFrameProcessorTestRunner.endFrameProcessingAndGetImage();
videoFrameProcessorTestRunner.queueInputBitmap(
readBitmap(ORIGINAL_PNG_ASSET_PATH),
/* durationUs= */ 2 * C.MICROS_PER_SECOND,
/* frameRate= */ 3);
int actualFrameCount = framesProduced.get();
assertThat(actualFrameCount).isEqualTo(/* expected= */ 2);
}
private VideoFrameProcessorTestRunner.Builder getDefaultFrameProcessorTestRunnerBuilder(
String testId) {
return new VideoFrameProcessorTestRunner.Builder()
.setTestId(testId)
.setVideoFrameProcessorFactory(new DefaultVideoFrameProcessor.Factory())
.setIsInputTextureExternal(false)
.setOnFrameAvailableListener((unused) -> checkNotNull(framesProduced).incrementAndGet());
}
}
library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessorPixelTest.java
...@@ -27,6 +27,7 @@ import android.content.Context; ...@@ -27,6 +27,7 @@ import android.content.Context;
import android.graphics.Bitmap; import android.graphics.Bitmap;
import android.graphics.Matrix; import android.graphics.Matrix;
import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.testutil.BitmapPixelTestUtil; import com.google.android.exoplayer2.testutil.BitmapPixelTestUtil;
import com.google.android.exoplayer2.testutil.VideoFrameProcessorTestRunner; import com.google.android.exoplayer2.testutil.VideoFrameProcessorTestRunner;
import com.google.android.exoplayer2.util.Effect; import com.google.android.exoplayer2.util.Effect;
...@@ -109,7 +110,9 @@ public final class DefaultVideoFrameProcessorPixelTest { ...@@ -109,7 +110,9 @@ public final class DefaultVideoFrameProcessorPixelTest {
getDefaultFrameProcessorTestRunnerBuilder(testId).setIsInputTextureExternal(false).build(); getDefaultFrameProcessorTestRunnerBuilder(testId).setIsInputTextureExternal(false).build();
Bitmap expectedBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH); Bitmap expectedBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH);
Bitmap actualBitmap = videoFrameProcessorTestRunner.processImageFrameAndEnd(expectedBitmap); videoFrameProcessorTestRunner.queueInputBitmap(
expectedBitmap, C.MICROS_PER_SECOND, /* frameRate= */ 1);
Bitmap actualBitmap = videoFrameProcessorTestRunner.endFrameProcessingAndGetImage();
// TODO(b/207848601): Switch to using proper tooling for testing against golden data. // TODO(b/207848601): Switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference = float averagePixelAbsoluteDifference =
...@@ -134,16 +137,15 @@ public final class DefaultVideoFrameProcessorPixelTest { ...@@ -134,16 +137,15 @@ public final class DefaultVideoFrameProcessorPixelTest {
Bitmap originalBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH); Bitmap originalBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH);
Bitmap expectedBitmap = readBitmap(WRAPPED_CROP_PNG_ASSET_PATH); Bitmap expectedBitmap = readBitmap(WRAPPED_CROP_PNG_ASSET_PATH);
Bitmap actualBitmap = videoFrameProcessorTestRunner.processImageFrameAndEnd(originalBitmap); videoFrameProcessorTestRunner.queueInputBitmap(
originalBitmap, C.MICROS_PER_SECOND, /* frameRate= */ 1);
Bitmap actualBitmap = videoFrameProcessorTestRunner.endFrameProcessingAndGetImage();
// TODO(b/207848601): Switch to using proper tooling for testing against golden data. // TODO(b/207848601): Switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference = float averagePixelAbsoluteDifference =
getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId); getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
} }
// TODO(b/262693274): Once texture deletion is added to InternalTextureManager.java, add a test
// queuing multiple input bitmaps to ensure the operation successfully completes and that the
// correct number of frames has been queued.
@Test @Test
public void noEffects_withFrameCache_matchesGoldenFile() throws Exception { public void noEffects_withFrameCache_matchesGoldenFile() throws Exception {
......
library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessorVideoFrameReleaseTest.java
...@@ -273,14 +273,14 @@ public final class DefaultVideoFrameProcessorVideoFrameReleaseTest { ...@@ -273,14 +273,14 @@ public final class DefaultVideoFrameProcessorVideoFrameReleaseTest {
assertThat(actualReleaseTimesNs).isEqualTo(releaseTimesNs); assertThat(actualReleaseTimesNs).isEqualTo(releaseTimesNs);
} }
private interface OnFrameAvailableListener { private interface OnOutputFrameAvailableListener {
void onFrameAvailable(long presentationTimeUs); void onFrameAvailable(long presentationTimeUs);
} }
@EnsuresNonNull("defaultVideoFrameProcessor") @EnsuresNonNull("defaultVideoFrameProcessor")
private void processFramesToEndOfStream( private void processFramesToEndOfStream(
long[] inputPresentationTimesUs, long[] inputPresentationTimesUs,
OnFrameAvailableListener onFrameAvailableListener, OnOutputFrameAvailableListener onFrameAvailableListener,
boolean releaseFramesAutomatically) boolean releaseFramesAutomatically)
throws Exception { throws Exception {
AtomicReference<@NullableType VideoFrameProcessingException> AtomicReference<@NullableType VideoFrameProcessingException>
......
testutils/src/main/java/com/google/android/exoplayer2/testutil/VideoFrameProcessorTestRunner.java
...@@ -27,8 +27,8 @@ import android.graphics.PixelFormat; ...@@ -27,8 +27,8 @@ import android.graphics.PixelFormat;
import android.media.Image; import android.media.Image;
import android.media.ImageReader; import android.media.ImageReader;
import android.media.MediaFormat; import android.media.MediaFormat;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi; import androidx.annotation.RequiresApi;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.util.DebugViewProvider; import com.google.android.exoplayer2.util.DebugViewProvider;
import com.google.android.exoplayer2.util.Effect; import com.google.android.exoplayer2.util.Effect;
import com.google.android.exoplayer2.util.FrameInfo; import com.google.android.exoplayer2.util.FrameInfo;
...@@ -61,11 +61,13 @@ public final class VideoFrameProcessorTestRunner { ...@@ -61,11 +61,13 @@ public final class VideoFrameProcessorTestRunner {
private @MonotonicNonNull ColorInfo inputColorInfo; private @MonotonicNonNull ColorInfo inputColorInfo;
private @MonotonicNonNull ColorInfo outputColorInfo; private @MonotonicNonNull ColorInfo outputColorInfo;
private boolean isInputTextureExternal; private boolean isInputTextureExternal;
private OnOutputFrameAvailableListener onOutputFrameAvailableListener;
/** Creates a new instance with default values. */ /** Creates a new instance with default values. */
public Builder() { public Builder() {
pixelWidthHeightRatio = DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO; pixelWidthHeightRatio = DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO;
isInputTextureExternal = true; isInputTextureExternal = true;
onOutputFrameAvailableListener = (unused) -> {};
} }
/** /**
...@@ -94,7 +96,7 @@ public final class VideoFrameProcessorTestRunner { ...@@ -94,7 +96,7 @@ public final class VideoFrameProcessorTestRunner {
/** /**
* Sets the input video asset path. * Sets the input video asset path.
* *
* <p>This is a required value. * <p>No default value is set. Must be set when the input is a video file.
*/ */
@CanIgnoreReturnValue @CanIgnoreReturnValue
public Builder setVideoAssetPath(String videoAssetPath) { public Builder setVideoAssetPath(String videoAssetPath) {
...@@ -170,7 +172,8 @@ public final class VideoFrameProcessorTestRunner { ...@@ -170,7 +172,8 @@ public final class VideoFrameProcessorTestRunner {
return this; return this;
} }
/** /**
* Sets the input track type. See {@link VideoFrameProcessor.Factory#create}. * Sets whether input comes from an external texture. See {@link
* VideoFrameProcessor.Factory#create}.
* *
* <p>The default value is {@code true}. * <p>The default value is {@code true}.
*/ */
...@@ -180,10 +183,21 @@ public final class VideoFrameProcessorTestRunner { ...@@ -180,10 +183,21 @@ public final class VideoFrameProcessorTestRunner {
return this; return this;
} }
/**
* Sets the method to be called in {@link VideoFrameProcessor.Listener#onOutputFrameAvailable}.
*
* <p>The default value is a no-op.
*/
@CanIgnoreReturnValue
public Builder setOnFrameAvailableListener(
OnOutputFrameAvailableListener onOutputFrameAvailableListener) {
this.onOutputFrameAvailableListener = onOutputFrameAvailableListener;
return this;
}
public VideoFrameProcessorTestRunner build() throws VideoFrameProcessingException { public VideoFrameProcessorTestRunner build() throws VideoFrameProcessingException {
checkStateNotNull(testId, "testId must be set."); checkStateNotNull(testId, "testId must be set.");
checkStateNotNull(videoFrameProcessorFactory, "videoFrameProcessorFactory must be set."); checkStateNotNull(videoFrameProcessorFactory, "videoFrameProcessorFactory must be set.");
checkStateNotNull(videoAssetPath, "videoAssetPath must be set.");
return new VideoFrameProcessorTestRunner( return new VideoFrameProcessorTestRunner(
testId, testId,
...@@ -194,7 +208,8 @@ public final class VideoFrameProcessorTestRunner { ...@@ -194,7 +208,8 @@ public final class VideoFrameProcessorTestRunner {
pixelWidthHeightRatio, pixelWidthHeightRatio,
inputColorInfo == null ? ColorInfo.SDR_BT709_LIMITED : inputColorInfo, inputColorInfo == null ? ColorInfo.SDR_BT709_LIMITED : inputColorInfo,
outputColorInfo == null ? ColorInfo.SDR_BT709_LIMITED : outputColorInfo, outputColorInfo == null ? ColorInfo.SDR_BT709_LIMITED : outputColorInfo,
isInputTextureExternal); isInputTextureExternal,
onOutputFrameAvailableListener);
} }
} }
...@@ -205,7 +220,7 @@ public final class VideoFrameProcessorTestRunner { ...@@ -205,7 +220,7 @@ public final class VideoFrameProcessorTestRunner {
private static final int VIDEO_FRAME_PROCESSING_WAIT_MS = 5000; private static final int VIDEO_FRAME_PROCESSING_WAIT_MS = 5000;
private final String testId; private final String testId;
private final String videoAssetPath; private final @MonotonicNonNull String videoAssetPath;
private final String outputFileLabel; private final String outputFileLabel;
private final float pixelWidthHeightRatio; private final float pixelWidthHeightRatio;
private final AtomicReference<VideoFrameProcessingException> videoFrameProcessingException; private final AtomicReference<VideoFrameProcessingException> videoFrameProcessingException;
...@@ -218,13 +233,14 @@ public final class VideoFrameProcessorTestRunner { ...@@ -218,13 +233,14 @@ public final class VideoFrameProcessorTestRunner {
private VideoFrameProcessorTestRunner( private VideoFrameProcessorTestRunner(
String testId, String testId,
VideoFrameProcessor.Factory videoFrameProcessorFactory, VideoFrameProcessor.Factory videoFrameProcessorFactory,
String videoAssetPath, @Nullable String videoAssetPath,
String outputFileLabel, String outputFileLabel,
ImmutableList<Effect> effects, ImmutableList<Effect> effects,
float pixelWidthHeightRatio, float pixelWidthHeightRatio,
ColorInfo inputColorInfo, ColorInfo inputColorInfo,
ColorInfo outputColorInfo, ColorInfo outputColorInfo,
boolean isInputTextureExternal) boolean isInputTextureExternal,
OnOutputFrameAvailableListener onOutputFrameAvailableListener)
throws VideoFrameProcessingException { throws VideoFrameProcessingException {
this.testId = testId; this.testId = testId;
this.videoAssetPath = videoAssetPath; this.videoAssetPath = videoAssetPath;
...@@ -256,6 +272,7 @@ public final class VideoFrameProcessorTestRunner { ...@@ -256,6 +272,7 @@ public final class VideoFrameProcessorTestRunner {
@Override @Override
public void onOutputFrameAvailable(long presentationTimeUs) { public void onOutputFrameAvailable(long presentationTimeUs) {
// Do nothing as frames are released automatically. // Do nothing as frames are released automatically.
onOutputFrameAvailableListener.onFrameAvailable(presentationTimeUs);
} }
@Override @Override
...@@ -272,7 +289,7 @@ public final class VideoFrameProcessorTestRunner { ...@@ -272,7 +289,7 @@ public final class VideoFrameProcessorTestRunner {
public Bitmap processFirstFrameAndEnd() throws Exception { public Bitmap processFirstFrameAndEnd() throws Exception {
DecodeOneFrameUtil.decodeOneAssetFileFrame( DecodeOneFrameUtil.decodeOneAssetFileFrame(
videoAssetPath, checkNotNull(videoAssetPath),
new DecodeOneFrameUtil.Listener() { new DecodeOneFrameUtil.Listener() {
@Override @Override
public void onContainerExtracted(MediaFormat mediaFormat) { public void onContainerExtracted(MediaFormat mediaFormat) {
...@@ -294,16 +311,15 @@ public final class VideoFrameProcessorTestRunner { ...@@ -294,16 +311,15 @@ public final class VideoFrameProcessorTestRunner {
return endFrameProcessingAndGetImage(); return endFrameProcessingAndGetImage();
} }
public Bitmap processImageFrameAndEnd(Bitmap inputBitmap) throws Exception { public void queueInputBitmap(Bitmap inputBitmap, long durationUs, float frameRate) {
videoFrameProcessor.setInputFrameInfo( videoFrameProcessor.setInputFrameInfo(
new FrameInfo.Builder(inputBitmap.getWidth(), inputBitmap.getHeight()) new FrameInfo.Builder(inputBitmap.getWidth(), inputBitmap.getHeight())
.setPixelWidthHeightRatio(pixelWidthHeightRatio) .setPixelWidthHeightRatio(pixelWidthHeightRatio)
.build()); .build());
videoFrameProcessor.queueInputBitmap(inputBitmap, C.MICROS_PER_SECOND, /* frameRate= */ 1); videoFrameProcessor.queueInputBitmap(inputBitmap, durationUs, frameRate);
return endFrameProcessingAndGetImage();
} }
private Bitmap endFrameProcessingAndGetImage() throws Exception { public Bitmap endFrameProcessingAndGetImage() throws Exception {
videoFrameProcessor.signalEndOfInput(); videoFrameProcessor.signalEndOfInput();
Thread.sleep(VIDEO_FRAME_PROCESSING_WAIT_MS); Thread.sleep(VIDEO_FRAME_PROCESSING_WAIT_MS);
...@@ -322,4 +338,8 @@ public final class VideoFrameProcessorTestRunner { ...@@ -322,4 +338,8 @@ public final class VideoFrameProcessorTestRunner {
videoFrameProcessor.release(); videoFrameProcessor.release();
} }
} }
public interface OnOutputFrameAvailableListener {
void onFrameAvailable(long presentationTimeUs);
}
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment