Skip to content

SDK / exoplayer

Go to a project
Commit 0b1a904c by huangdarwin Committed by christosts
Options

HDR: Add device checks and mh for GL tone mapping PixelTests. 

Add checks to GL tone-mapping pixel tests, to ensure the device's decoder, API
version, and OpenGL implementation support GL tone-mapping before attempting it.

These tests should be run on mobile harness, to detect per-device failures, and
so are moved to transforemr/mh. Per b/263395272, these tests should ultimately
be in an effect/mh directory.

PiperOrigin-RevId: 505749974
parent 4fb651b1
Inline Side-by-side
Showing with 264 additions and 82 deletions
libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorPixelTest.java
...@@ -25,8 +25,6 @@ import static com.google.common.truth.Truth.assertThat; ...@@ -25,8 +25,6 @@ import static com.google.common.truth.Truth.assertThat;
import android.content.Context; import android.content.Context;
import android.graphics.Bitmap; import android.graphics.Bitmap;
import android.graphics.Matrix; import android.graphics.Matrix;
import androidx.media3.common.C;
import androidx.media3.common.ColorInfo;
import androidx.media3.common.Effect; import androidx.media3.common.Effect;
import androidx.media3.common.FrameProcessingException; import androidx.media3.common.FrameProcessingException;
import androidx.media3.common.util.Size; import androidx.media3.common.util.Size;
...@@ -36,7 +34,6 @@ import androidx.test.ext.junit.runners.AndroidJUnit4; ...@@ -36,7 +34,6 @@ import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.junit.After; import org.junit.After;
import org.junit.Ignore;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
...@@ -75,21 +72,9 @@ public final class GlEffectsFrameProcessorPixelTest { ...@@ -75,21 +72,9 @@ public final class GlEffectsFrameProcessorPixelTest {
"media/bitmap/sample_mp4_first_frame/electrical_colors/increase_brightness.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/increase_brightness.png";
public static final String GRAYSCALE_THEN_INCREASE_RED_CHANNEL_PNG_ASSET_PATH = public static final String GRAYSCALE_THEN_INCREASE_RED_CHANNEL_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/electrical_colors/grayscale_then_increase_red_channel.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/grayscale_then_increase_red_channel.png";
// This file is generated on a Pixel 7, because the emulator isn't able to decode HLG to generate
// this file.
public static final String TONE_MAP_HLG_TO_SDR_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/electrical_colors/tone_map_hlg_to_sdr.png";
// This file is generated on a Pixel 7, because the emulator isn't able to decode PQ to generate
// this file.
public static final String TONE_MAP_PQ_TO_SDR_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/electrical_colors/tone_map_pq_to_sdr.png";
/** Input video of which we only use the first frame. */ /** Input video of which we only use the first frame. */
private static final String INPUT_SDR_MP4_ASSET_STRING = "media/mp4/sample.mp4"; private static final String INPUT_SDR_MP4_ASSET_STRING = "media/mp4/sample.mp4";
/** Input HLG video of which we only use the first frame. */
private static final String INPUT_HLG_MP4_ASSET_STRING = "media/mp4/hlg-1080p.mp4";
/** Input PQ video of which we only use the first frame. */
private static final String INPUT_PQ_MP4_ASSET_STRING = "media/mp4/hdr10-1080p.mp4";
private @MonotonicNonNull FrameProcessorTestRunner frameProcessorTestRunner; private @MonotonicNonNull FrameProcessorTestRunner frameProcessorTestRunner;
...@@ -445,76 +430,11 @@ public final class GlEffectsFrameProcessorPixelTest { ...@@ -445,76 +430,11 @@ public final class GlEffectsFrameProcessorPixelTest {
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
} }
@Test
@Ignore("b/261877288 Test can only run on physical devices because decoder can't decode HLG.")
public void toneMap_hlgFrame_matchesGoldenFile() throws Exception {
// TODO(b/239735341): Move this test to mobileharness testing.
String testId = "toneMap_hlgFrame_matchesGoldenFile";
ColorInfo hlgColor =
new ColorInfo.Builder()
.setColorSpace(C.COLOR_SPACE_BT2020)
.setColorRange(C.COLOR_RANGE_LIMITED)
.setColorTransfer(C.COLOR_TRANSFER_HLG)
.build();
ColorInfo toneMapSdrColor =
new ColorInfo.Builder()
.setColorSpace(C.COLOR_SPACE_BT709)
.setColorRange(C.COLOR_RANGE_LIMITED)
.setColorTransfer(C.COLOR_TRANSFER_GAMMA_2_2)
.build();
frameProcessorTestRunner =
getDefaultFrameProcessorTestRunnerBuilder(testId)
.setVideoAssetPath(INPUT_HLG_MP4_ASSET_STRING)
.setInputColorInfo(hlgColor)
.setOutputColorInfo(toneMapSdrColor)
.build();
Bitmap expectedBitmap = readBitmap(TONE_MAP_HLG_TO_SDR_PNG_ASSET_PATH);
Bitmap actualBitmap = frameProcessorTestRunner.processFirstFrameAndEnd();
// TODO(b/207848601): switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference =
getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
@Test
@Ignore("b/261877288 Test can only run on physical devices because decoder can't decode PQ.")
public void toneMap_pqFrame_matchesGoldenFile() throws Exception {
// TODO(b/239735341): Move this test to mobileharness testing.
String testId = "toneMap_pqFrame_matchesGoldenFile";
ColorInfo pqColor =
new ColorInfo.Builder()
.setColorSpace(C.COLOR_SPACE_BT2020)
.setColorRange(C.COLOR_RANGE_LIMITED)
.setColorTransfer(C.COLOR_TRANSFER_ST2084)
.build();
ColorInfo toneMapSdrColor =
new ColorInfo.Builder()
.setColorSpace(C.COLOR_SPACE_BT709)
.setColorRange(C.COLOR_RANGE_LIMITED)
.setColorTransfer(C.COLOR_TRANSFER_GAMMA_2_2)
.build();
frameProcessorTestRunner =
getDefaultFrameProcessorTestRunnerBuilder(testId)
.setVideoAssetPath(INPUT_PQ_MP4_ASSET_STRING)
.setInputColorInfo(pqColor)
.setOutputColorInfo(toneMapSdrColor)
.build();
Bitmap expectedBitmap = readBitmap(TONE_MAP_PQ_TO_SDR_PNG_ASSET_PATH);
Bitmap actualBitmap = frameProcessorTestRunner.processFirstFrameAndEnd();
// TODO(b/207848601): switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference =
getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
// TODO(b/227624622): Add a test for HDR input after BitmapPixelTestUtil can read HDR bitmaps, // TODO(b/227624622): Add a test for HDR input after BitmapPixelTestUtil can read HDR bitmaps,
// using GlEffectWrapper to ensure usage of intermediate textures. // using GlEffectWrapper to ensure usage of intermediate textures.
public FrameProcessorTestRunner.Builder getDefaultFrameProcessorTestRunnerBuilder(String testId) { private FrameProcessorTestRunner.Builder getDefaultFrameProcessorTestRunnerBuilder(
String testId) {
return new FrameProcessorTestRunner.Builder() return new FrameProcessorTestRunner.Builder()
.setTestId(testId) .setTestId(testId)
.setFrameProcessorFactory(new GlEffectsFrameProcessor.Factory()) .setFrameProcessorFactory(new GlEffectsFrameProcessor.Factory())
......
libraries/test_utils/src/main/java/androidx/media3/test/utils/DecodeOneFrameUtil.java
...@@ -20,22 +20,29 @@ import static androidx.media3.common.util.Assertions.checkNotNull; ...@@ -20,22 +20,29 @@ import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkStateNotNull; import static androidx.media3.common.util.Assertions.checkStateNotNull;
import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertThat;
import static java.lang.Math.round;
import android.content.Context; import android.content.Context;
import android.content.res.AssetFileDescriptor; import android.content.res.AssetFileDescriptor;
import android.graphics.SurfaceTexture; import android.graphics.SurfaceTexture;
import android.media.MediaCodec; import android.media.MediaCodec;
import android.media.MediaCodecList;
import android.media.MediaExtractor; import android.media.MediaExtractor;
import android.media.MediaFormat; import android.media.MediaFormat;
import android.view.Surface; import android.view.Surface;
import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes; import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.MediaFormatUtil;
import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.Nullable;
/** Utilities for decoding a frame for tests. */ /** Utilities for decoding a frame for tests. */
@UnstableApi @UnstableApi
public class DecodeOneFrameUtil { public class DecodeOneFrameUtil {
public static final String NO_DECODER_SUPPORT_ERROR_STRING =
"No MediaCodec decoders on this device support this value.";
/** Listener for decoding events. */ /** Listener for decoding events. */
public interface Listener { public interface Listener {
...@@ -126,9 +133,13 @@ public class DecodeOneFrameUtil { ...@@ -126,9 +133,13 @@ public class DecodeOneFrameUtil {
} }
checkStateNotNull(mediaFormat); checkStateNotNull(mediaFormat);
if (!isSupportedByDecoder(mediaFormat)) {
throw new UnsupportedOperationException(NO_DECODER_SUPPORT_ERROR_STRING);
}
// Queue the first video frame from the extractor. // Queue the first video frame from the extractor.
String mimeType = checkNotNull(mediaFormat.getString(MediaFormat.KEY_MIME)); String mimeType = checkNotNull(mediaFormat.getString(MediaFormat.KEY_MIME));
mediaCodec = MediaCodec.createDecoderByType(mimeType); mediaCodec = MediaCodec.createDecoderByType(mimeType);
mediaCodec.configure(mediaFormat, surface, /* crypto= */ null, /* flags= */ 0); mediaCodec.configure(mediaFormat, surface, /* crypto= */ null, /* flags= */ 0);
mediaCodec.start(); mediaCodec.start();
int inputBufferIndex = mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US); int inputBufferIndex = mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US);
...@@ -173,5 +184,38 @@ public class DecodeOneFrameUtil { ...@@ -173,5 +184,38 @@ public class DecodeOneFrameUtil {
} }
} }
/**
* Returns whether a decoder supports this {@link MediaFormat}.
*
* <p>Capability check is similar to
* androidx.media3.transformer.EncoderUtil.java#findCodecForFormat().
*/
private static boolean isSupportedByDecoder(MediaFormat format) {
if (Util.SDK_INT < 21) {
throw new UnsupportedOperationException("Unable to detect decoder support under API 21.");
}
// TODO(b/266923205): De-duplicate logic from EncoderUtil.java#findCodecForFormat().
MediaCodecList mediaCodecList = new MediaCodecList(MediaCodecList.REGULAR_CODECS);
// Format must not include KEY_FRAME_RATE on API21.
// https://developer.android.com/reference/android/media/MediaCodecList#findDecoderForFormat(android.media.MediaFormat)
float frameRate = Format.NO_VALUE;
if (Util.SDK_INT == 21 && format.containsKey(MediaFormat.KEY_FRAME_RATE)) {
try {
frameRate = format.getFloat(MediaFormat.KEY_FRAME_RATE);
} catch (ClassCastException e) {
frameRate = format.getInteger(MediaFormat.KEY_FRAME_RATE);
}
// Clears the frame rate field.
format.setString(MediaFormat.KEY_FRAME_RATE, null);
}
@Nullable String mediaCodecName = mediaCodecList.findDecoderForFormat(format);
if (Util.SDK_INT == 21) {
MediaFormatUtil.maybeSetInteger(format, MediaFormat.KEY_FRAME_RATE, round(frameRate));
}
return mediaCodecName != null;
}
private DecodeOneFrameUtil() {} private DecodeOneFrameUtil() {}
} }
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ToneMapHdrToSdrUsingOpenGlPixelTest.java 0 → 100644
/*
* Copyright 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer.mh;
import static androidx.media3.test.utils.BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888;
import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_1080P_4_SECOND_HDR10_FORMAT;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_1080P_5_SECOND_HLG10_FORMAT;
import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped;
import static androidx.media3.transformer.AndroidTestUtil.skipAndLogIfInsufficientCodecSupport;
import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
import static com.google.common.truth.Truth.assertThat;
import android.graphics.Bitmap;
import android.util.Log;
import androidx.media3.common.C;
import androidx.media3.common.ColorInfo;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.Util;
import androidx.media3.effect.GlEffectsFrameProcessor;
import androidx.media3.test.utils.DecodeOneFrameUtil;
import androidx.media3.test.utils.FrameProcessorTestRunner;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.junit.After;
import org.junit.Test;
import org.junit.runner.RunWith;
/**
* Instrumentation pixel-test for HDR to SDR tone-mapping via {@link GlEffectsFrameProcessor}.
*
* <p>Uses a {@link GlEffectsFrameProcessor} to process one frame, and checks that the actual output
* matches expected output, either from a golden file or from another edit.
*/
// TODO(b/263395272): Move this test to effects/mh tests.
@RunWith(AndroidJUnit4.class)
public final class ToneMapHdrToSdrUsingOpenGlPixelTest {
private static final String TAG = "ToneMapHdrToSdrGl";
/**
* Maximum allowed average pixel difference between the expected and actual edited images in
* on-device pixel difference-based tests. The value is chosen so that differences in behavior
* across codec/OpenGL versions don't affect whether the test passes for most devices, but
* substantial distortions introduced by changes in tested components will cause the test to fail.
*/
private static final float MAXIMUM_DEVICE_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE = 5f;
// This file is generated on a Pixel 7, because the emulator isn't able to decode HLG to generate
// this file.
private static final String TONE_MAP_HLG_TO_SDR_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/electrical_colors/tone_map_hlg_to_sdr.png";
// This file is generated on a Pixel 7, because the emulator isn't able to decode PQ to generate
// this file.
private static final String TONE_MAP_PQ_TO_SDR_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/electrical_colors/tone_map_pq_to_sdr.png";
/** Input HLG video of which we only use the first frame. */
private static final String INPUT_HLG_MP4_ASSET_STRING = "media/mp4/hlg-1080p.mp4";
/** Input PQ video of which we only use the first frame. */
private static final String INPUT_PQ_MP4_ASSET_STRING = "media/mp4/hdr10-1080p.mp4";
private static final String SKIP_REASON_NO_OPENGL_UNDER_API_29 =
"OpenGL-based HDR to SDR tone mapping is unsupported below API 29.";
private static final String SKIP_REASON_NO_YUV = "Device lacks YUV extension support.";
private @MonotonicNonNull FrameProcessorTestRunner frameProcessorTestRunner;
@After
public void release() {
if (frameProcessorTestRunner != null) {
frameProcessorTestRunner.release();
}
}
@Test
public void toneMap_hlgFrame_matchesGoldenFile() throws Exception {
String testId = "toneMap_hlgFrame_matchesGoldenFile";
if (Util.SDK_INT < 29) {
recordTestSkipped(getApplicationContext(), testId, SKIP_REASON_NO_OPENGL_UNDER_API_29);
return;
}
if (!GlUtil.isYuvTargetExtensionSupported()) {
recordTestSkipped(getApplicationContext(), testId, SKIP_REASON_NO_YUV);
return;
}
if (skipAndLogIfInsufficientCodecSupport(
getApplicationContext(),
testId,
/* decodingFormat= */ MP4_ASSET_1080P_5_SECOND_HLG10_FORMAT,
/* encodingFormat= */ null)) {
return;
}
ColorInfo hlgColor =
new ColorInfo.Builder()
.setColorSpace(C.COLOR_SPACE_BT2020)
.setColorRange(C.COLOR_RANGE_LIMITED)
.setColorTransfer(C.COLOR_TRANSFER_HLG)
.build();
ColorInfo toneMapSdrColor =
new ColorInfo.Builder()
.setColorSpace(C.COLOR_SPACE_BT709)
.setColorRange(C.COLOR_RANGE_LIMITED)
.setColorTransfer(C.COLOR_TRANSFER_GAMMA_2_2)
.build();
frameProcessorTestRunner =
getDefaultFrameProcessorTestRunnerBuilder(testId)
.setVideoAssetPath(INPUT_HLG_MP4_ASSET_STRING)
.setInputColorInfo(hlgColor)
.setOutputColorInfo(toneMapSdrColor)
.build();
Bitmap expectedBitmap = readBitmap(TONE_MAP_HLG_TO_SDR_PNG_ASSET_PATH);
Bitmap actualBitmap;
try {
actualBitmap = frameProcessorTestRunner.processFirstFrameAndEnd();
} catch (UnsupportedOperationException e) {
if (e.getMessage() != null
&& e.getMessage().equals(DecodeOneFrameUtil.NO_DECODER_SUPPORT_ERROR_STRING)) {
recordTestSkipped(
getApplicationContext(),
testId,
/* reason= */ DecodeOneFrameUtil.NO_DECODER_SUPPORT_ERROR_STRING);
return;
} else {
throw e;
}
}
Log.i(TAG, "Successfully tone mapped.");
// TODO(b/207848601): switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference =
getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
assertThat(averagePixelAbsoluteDifference)
.isAtMost(MAXIMUM_DEVICE_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
@Test
public void toneMap_pqFrame_matchesGoldenFile() throws Exception {
// TODO(b/239735341): Move this test to mobileharness testing.
String testId = "toneMap_pqFrame_matchesGoldenFile";
if (Util.SDK_INT < 29) {
recordTestSkipped(getApplicationContext(), testId, SKIP_REASON_NO_OPENGL_UNDER_API_29);
return;
}
if (!GlUtil.isYuvTargetExtensionSupported()) {
recordTestSkipped(getApplicationContext(), testId, SKIP_REASON_NO_YUV);
return;
}
if (skipAndLogIfInsufficientCodecSupport(
getApplicationContext(),
testId,
/* decodingFormat= */ MP4_ASSET_1080P_4_SECOND_HDR10_FORMAT,
/* encodingFormat= */ null)) {
return;
}
ColorInfo pqColor =
new ColorInfo.Builder()
.setColorSpace(C.COLOR_SPACE_BT2020)
.setColorRange(C.COLOR_RANGE_LIMITED)
.setColorTransfer(C.COLOR_TRANSFER_ST2084)
.build();
ColorInfo toneMapSdrColor =
new ColorInfo.Builder()
.setColorSpace(C.COLOR_SPACE_BT709)
.setColorRange(C.COLOR_RANGE_LIMITED)
.setColorTransfer(C.COLOR_TRANSFER_GAMMA_2_2)
.build();
frameProcessorTestRunner =
getDefaultFrameProcessorTestRunnerBuilder(testId)
.setVideoAssetPath(INPUT_PQ_MP4_ASSET_STRING)
.setInputColorInfo(pqColor)
.setOutputColorInfo(toneMapSdrColor)
.build();
Bitmap expectedBitmap = readBitmap(TONE_MAP_PQ_TO_SDR_PNG_ASSET_PATH);
Bitmap actualBitmap;
try {
actualBitmap = frameProcessorTestRunner.processFirstFrameAndEnd();
} catch (UnsupportedOperationException e) {
if (e.getMessage() != null
&& e.getMessage().equals(DecodeOneFrameUtil.NO_DECODER_SUPPORT_ERROR_STRING)) {
recordTestSkipped(
getApplicationContext(),
testId,
/* reason= */ DecodeOneFrameUtil.NO_DECODER_SUPPORT_ERROR_STRING);
return;
} else {
throw e;
}
}
Log.i(TAG, "Successfully tone mapped.");
// TODO(b/207848601): switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference =
getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
assertThat(averagePixelAbsoluteDifference)
.isAtMost(MAXIMUM_DEVICE_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
private FrameProcessorTestRunner.Builder getDefaultFrameProcessorTestRunnerBuilder(
String testId) {
return new FrameProcessorTestRunner.Builder()
.setTestId(testId)
.setFrameProcessorFactory(new GlEffectsFrameProcessor.Factory());
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment