Skip to content

SDK / exoplayer

Go to a project
Commit a534f60f by leonwind Committed by microkatz
Options

Add OETF and EOTF ES2 fragment shaders for non-HDR frames. 

* Transform the intermediate color space to linear SDR by applying the SMPTE 170M EOTF and OETF.
* Use linear colors for the color filter pixel tests and update all golden bitmaps.

PiperOrigin-RevId: 476124592
(cherry picked from commit afd829e5)
parent eb496dab
Inline Side-by-side
Showing with 180 additions and 66 deletions
google3/third_party/java_src/android_libs/media/libraries/effect/src/androidTest/java/androidx/media3/effect/ContrastPixelTest.java
...@@ -48,13 +48,13 @@ import org.junit.runner.RunWith; ...@@ -48,13 +48,13 @@ import org.junit.runner.RunWith;
@RunWith(AndroidJUnit4.class) @RunWith(AndroidJUnit4.class)
public class ContrastPixelTest { public class ContrastPixelTest {
public static final String ORIGINAL_PNG_ASSET_PATH = public static final String ORIGINAL_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/original.png"; "media/bitmap/sample_mp4_first_frame/linear_colors/original.png";
public static final String INCREASE_CONTRAST_PNG_ASSET_PATH = public static final String INCREASE_CONTRAST_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/increase_contrast.png"; "media/bitmap/sample_mp4_first_frame/linear_colors/increase_contrast.png";
public static final String DECREASE_CONTRAST_PNG_ASSET_PATH = public static final String DECREASE_CONTRAST_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/decrease_contrast.png"; "media/bitmap/sample_mp4_first_frame/linear_colors/decrease_contrast.png";
public static final String MAXIMUM_CONTRAST_PNG_ASSET_PATH = public static final String MAXIMUM_CONTRAST_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/maximum_contrast.png"; "media/bitmap/sample_mp4_first_frame/linear_colors/maximum_contrast.png";
// OpenGL uses floats in [0, 1] and maps 0.5f to 128 = 256 / 2. // OpenGL uses floats in [0, 1] and maps 0.5f to 128 = 256 / 2.
private static final int OPENGL_NEUTRAL_RGB_VALUE = 128; private static final int OPENGL_NEUTRAL_RGB_VALUE = 128;
......
google3/third_party/java_src/android_libs/media/libraries/effect/src/androidTest/java/androidx/media3/effect/CropPixelTest.java
...@@ -47,11 +47,11 @@ import org.junit.runner.RunWith; ...@@ -47,11 +47,11 @@ import org.junit.runner.RunWith;
@RunWith(AndroidJUnit4.class) @RunWith(AndroidJUnit4.class)
public final class CropPixelTest { public final class CropPixelTest {
public static final String ORIGINAL_PNG_ASSET_PATH = public static final String ORIGINAL_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/original.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/original.png";
public static final String CROP_SMALLER_PNG_ASSET_PATH = public static final String CROP_SMALLER_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/crop_smaller.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/crop_smaller.png";
public static final String CROP_LARGER_PNG_ASSET_PATH = public static final String CROP_LARGER_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/crop_larger.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/crop_larger.png";
private final Context context = getApplicationContext(); private final Context context = getApplicationContext();
......
google3/third_party/java_src/android_libs/media/libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorPixelTest.java
...@@ -58,27 +58,27 @@ import org.junit.runner.RunWith; ...@@ -58,27 +58,27 @@ import org.junit.runner.RunWith;
@RunWith(AndroidJUnit4.class) @RunWith(AndroidJUnit4.class)
public final class GlEffectsFrameProcessorPixelTest { public final class GlEffectsFrameProcessorPixelTest {
public static final String ORIGINAL_PNG_ASSET_PATH = public static final String ORIGINAL_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/original.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/original.png";
public static final String SCALE_WIDE_PNG_ASSET_PATH = public static final String SCALE_WIDE_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/scale_wide.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/scale_wide.png";
public static final String TRANSLATE_RIGHT_PNG_ASSET_PATH = public static final String TRANSLATE_RIGHT_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/translate_right.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/translate_right.png";
public static final String ROTATE_THEN_TRANSLATE_PNG_ASSET_PATH = public static final String ROTATE_THEN_TRANSLATE_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/rotate_then_translate.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/rotate_then_translate.png";
public static final String ROTATE_THEN_SCALE_PNG_ASSET_PATH = public static final String ROTATE_THEN_SCALE_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/rotate45_then_scale2w.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/rotate45_then_scale2w.png";
public static final String TRANSLATE_THEN_ROTATE_PNG_ASSET_PATH = public static final String TRANSLATE_THEN_ROTATE_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/translate_then_rotate.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/translate_then_rotate.png";
public static final String REQUEST_OUTPUT_HEIGHT_PNG_ASSET_PATH = public static final String REQUEST_OUTPUT_HEIGHT_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/request_output_height.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/request_output_height.png";
public static final String CROP_THEN_ASPECT_RATIO_PNG_ASSET_PATH = public static final String CROP_THEN_ASPECT_RATIO_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/crop_then_aspect_ratio.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/crop_then_aspect_ratio.png";
public static final String ROTATE45_SCALE_TO_FIT_PNG_ASSET_PATH = public static final String ROTATE45_SCALE_TO_FIT_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/rotate_45_scale_to_fit.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/rotate_45_scale_to_fit.png";
public static final String INCREASE_BRIGHTNESS_PNG_ASSET_PATH = public static final String INCREASE_BRIGHTNESS_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/increase_brightness.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/increase_brightness.png";
public static final String GRAYSCALE_THEN_INCREASE_RED_CHANNEL_PNG_ASSET_PATH = public static final String GRAYSCALE_THEN_INCREASE_RED_CHANNEL_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/grayscale_then_increase_red_channel.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/grayscale_then_increase_red_channel.png";
/** Input video of which we only use the first frame. */ /** Input video of which we only use the first frame. */
private static final String INPUT_MP4_ASSET_STRING = "media/mp4/sample.mp4"; private static final String INPUT_MP4_ASSET_STRING = "media/mp4/sample.mp4";
......
google3/third_party/java_src/android_libs/media/libraries/effect/src/androidTest/java/androidx/media3/effect/MatrixTextureProcessorPixelTest.java
...@@ -46,13 +46,13 @@ import org.junit.runner.RunWith; ...@@ -46,13 +46,13 @@ import org.junit.runner.RunWith;
@RunWith(AndroidJUnit4.class) @RunWith(AndroidJUnit4.class)
public final class MatrixTextureProcessorPixelTest { public final class MatrixTextureProcessorPixelTest {
public static final String ORIGINAL_PNG_ASSET_PATH = public static final String ORIGINAL_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/original.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/original.png";
public static final String TRANSLATE_RIGHT_PNG_ASSET_PATH = public static final String TRANSLATE_RIGHT_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/translate_right.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/translate_right.png";
public static final String SCALE_NARROW_PNG_ASSET_PATH = public static final String SCALE_NARROW_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/scale_narrow.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/scale_narrow.png";
public static final String ROTATE_90_PNG_ASSET_PATH = public static final String ROTATE_90_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/rotate90.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/rotate90.png";
private final Context context = getApplicationContext(); private final Context context = getApplicationContext();
......
google3/third_party/java_src/android_libs/media/libraries/effect/src/androidTest/java/androidx/media3/effect/PresentationPixelTest.java
...@@ -48,19 +48,19 @@ import org.junit.runner.RunWith; ...@@ -48,19 +48,19 @@ import org.junit.runner.RunWith;
@RunWith(AndroidJUnit4.class) @RunWith(AndroidJUnit4.class)
public final class PresentationPixelTest { public final class PresentationPixelTest {
public static final String ORIGINAL_PNG_ASSET_PATH = public static final String ORIGINAL_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/original.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/original.png";
public static final String ASPECT_RATIO_SCALE_TO_FIT_NARROW_PNG_ASSET_PATH = public static final String ASPECT_RATIO_SCALE_TO_FIT_NARROW_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/aspect_ratio_scale_to_fit_narrow.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/aspect_ratio_scale_to_fit_narrow.png";
public static final String ASPECT_RATIO_SCALE_TO_FIT_WIDE_PNG_ASSET_PATH = public static final String ASPECT_RATIO_SCALE_TO_FIT_WIDE_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/aspect_ratio_scale_to_fit_wide.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/aspect_ratio_scale_to_fit_wide.png";
public static final String ASPECT_RATIO_SCALE_TO_FIT_WITH_CROP_NARROW_PNG_ASSET_PATH = public static final String ASPECT_RATIO_SCALE_TO_FIT_WITH_CROP_NARROW_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/aspect_ratio_scale_to_fit_with_crop_narrow.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/aspect_ratio_scale_to_fit_with_crop_narrow.png";
public static final String ASPECT_RATIO_SCALE_TO_FIT_WITH_CROP_WIDE_PNG_ASSET_PATH = public static final String ASPECT_RATIO_SCALE_TO_FIT_WITH_CROP_WIDE_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/aspect_ratio_scale_to_fit_with_crop_wide.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/aspect_ratio_scale_to_fit_with_crop_wide.png";
public static final String ASPECT_RATIO_STRETCH_TO_FIT_NARROW_PNG_ASSET_PATH = public static final String ASPECT_RATIO_STRETCH_TO_FIT_NARROW_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/aspect_ratio_stretch_to_fit_narrow.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/aspect_ratio_stretch_to_fit_narrow.png";
public static final String ASPECT_RATIO_STRETCH_TO_FIT_WIDE_PNG_ASSET_PATH = public static final String ASPECT_RATIO_STRETCH_TO_FIT_WIDE_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/aspect_ratio_stretch_to_fit_wide.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/aspect_ratio_stretch_to_fit_wide.png";
private final Context context = getApplicationContext(); private final Context context = getApplicationContext();
......
google3/third_party/java_src/android_libs/media/libraries/effect/src/androidTest/java/androidx/media3/effect/RgbAdjustmentPixelTest.java
...@@ -50,13 +50,13 @@ import org.junit.runner.RunWith; ...@@ -50,13 +50,13 @@ import org.junit.runner.RunWith;
@RunWith(AndroidJUnit4.class) @RunWith(AndroidJUnit4.class)
public final class RgbAdjustmentPixelTest { public final class RgbAdjustmentPixelTest {
public static final String ORIGINAL_PNG_ASSET_PATH = public static final String ORIGINAL_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/original.png"; "media/bitmap/sample_mp4_first_frame/linear_colors/original.png";
public static final String ONLY_RED_CHANNEL_PNG_ASSET_PATH = public static final String ONLY_RED_CHANNEL_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/only_red_channel.png"; "media/bitmap/sample_mp4_first_frame/linear_colors/only_red_channel.png";
public static final String INCREASE_RED_CHANNEL_PNG_ASSET_PATH = public static final String INCREASE_RED_CHANNEL_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/increase_red_channel.png"; "media/bitmap/sample_mp4_first_frame/linear_colors/increase_red_channel.png";
public static final String INCREASE_BRIGHTNESS_PNG_ASSET_PATH = public static final String INCREASE_BRIGHTNESS_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/increase_brightness.png"; "media/bitmap/sample_mp4_first_frame/linear_colors/increase_brightness.png";
private final Context context = getApplicationContext(); private final Context context = getApplicationContext();
......
google3/third_party/java_src/android_libs/media/libraries/effect/src/androidTest/java/androidx/media3/effect/RgbFilterPixelTest.java
...@@ -48,11 +48,11 @@ import org.junit.runner.RunWith; ...@@ -48,11 +48,11 @@ import org.junit.runner.RunWith;
@RunWith(AndroidJUnit4.class) @RunWith(AndroidJUnit4.class)
public final class RgbFilterPixelTest { public final class RgbFilterPixelTest {
public static final String ORIGINAL_PNG_ASSET_PATH = public static final String ORIGINAL_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/original.png"; "media/bitmap/sample_mp4_first_frame/linear_colors/original.png";
public static final String GRAYSCALE_PNG_ASSET_PATH = public static final String GRAYSCALE_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/grayscale.png"; "media/bitmap/sample_mp4_first_frame/linear_colors/grayscale.png";
public static final String INVERT_PNG_ASSET_PATH = public static final String INVERT_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/invert.png"; "media/bitmap/sample_mp4_first_frame/linear_colors/invert.png";
private final Context context = getApplicationContext(); private final Context context = getApplicationContext();
......
google3/third_party/java_src/android_libs/media/libraries/effect/src/androidTest/java/androidx/media3/effect/SingleColorLutPixelTest.java
...@@ -48,13 +48,13 @@ import org.junit.runner.RunWith; ...@@ -48,13 +48,13 @@ import org.junit.runner.RunWith;
@RunWith(AndroidJUnit4.class) @RunWith(AndroidJUnit4.class)
public class SingleColorLutPixelTest { public class SingleColorLutPixelTest {
public static final String ORIGINAL_PNG_ASSET_PATH = public static final String ORIGINAL_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/original.png"; "media/bitmap/sample_mp4_first_frame/linear_colors/original.png";
public static final String LUT_MAP_WHITE_TO_GREEN_ASSET_PATH = public static final String LUT_MAP_WHITE_TO_GREEN_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/lut_map_white_to_green.png"; "media/bitmap/sample_mp4_first_frame/linear_colors/lut_map_white_to_green.png";
public static final String GRAYSCALE_PNG_ASSET_PATH = public static final String GRAYSCALE_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/grayscale.png"; "media/bitmap/sample_mp4_first_frame/linear_colors/grayscale.png";
public static final String INVERT_PNG_ASSET_PATH = public static final String INVERT_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/invert.png"; "media/bitmap/sample_mp4_first_frame/linear_colors/invert.png";
public static final String VERTICAL_HALD_IDENTITY_LUT = "media/bitmap/lut/identity.png"; public static final String VERTICAL_HALD_IDENTITY_LUT = "media/bitmap/lut/identity.png";
public static final String VERTICAL_HALD_GRAYSCALE_LUT = "media/bitmap/lut/grayscale.png"; public static final String VERTICAL_HALD_GRAYSCALE_LUT = "media/bitmap/lut/grayscale.png";
public static final String VERTICAL_HALD_INVERTED_LUT = "media/bitmap/lut/inverted.png"; public static final String VERTICAL_HALD_INVERTED_LUT = "media/bitmap/lut/inverted.png";
......
google3/third_party/java_src/android_libs/media/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_external_yuv_es3.glsl
...@@ -104,4 +104,5 @@ void main() { ...@@ -104,4 +104,5 @@ void main() {
vec3 srcYuv = texture(uTexSampler, vTexSamplingCoord).xyz; vec3 srcYuv = texture(uTexSampler, vTexSamplingCoord).xyz;
vec3 rgb = yuvToRgb(srcYuv); vec3 rgb = yuvToRgb(srcYuv);
outColor = uRgbMatrix * vec4(getOpticalColor(rgb), 1.0); outColor = uRgbMatrix * vec4(getOpticalColor(rgb), 1.0);
// TODO(b/241902517): Transform optical to electrical colors.
} }
google3/third_party/java_src/android_libs/media/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_external_es2.glsl 0 → 100644
#version 100
// Copyright 2021 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ES 2 fragment shader that:
// 1. Samples from an external texture with uTexSampler copying from this
// texture to the current output.
// 2. Transforms the electrical colors to optical colors using the SMPTE 170M
// EOTF.
// 3. Applies a 4x4 RGB color matrix to change the pixel colors.
// 4. Transforms the optical colors back to electrical ones if uApplyOetf == 1
// using the SMPTE 170M OETF.
#extension GL_OES_EGL_image_external : require
precision mediump float;
uniform samplerExternalOES uTexSampler;
uniform mat4 uRgbMatrix;
varying vec2 vTexSamplingCoord;
uniform int uApplyOetf;
const float inverseGamma = 0.4500;
const float gamma = 1.0 / inverseGamma;
// Transforms a single channel from electrical to optical SDR.
float sdrEotfSingleChannel(float electricalChannel) {
// Specification:
// https://www.itu.int/rec/R-REC-BT.1700-0-200502-I/en
return electricalChannel < 0.0812
? electricalChannel / 4.500
: pow((electricalChannel + 0.099) / 1.099, gamma);
}
// Transforms electronical to optical SDR using the SMPTE 170M EOTF.
vec3 sdrEotf(vec3 electricalColor) {
return vec3(
sdrEotfSingleChannel(electricalColor.r),
sdrEotfSingleChannel(electricalColor.g),
sdrEotfSingleChannel(electricalColor.b));
}
// Transforms a single channel from optical to electrical SDR.
float sdrOetfSingleChannel(float opticalChannel) {
// Specification:
// https://www.itu.int/rec/R-REC-BT.1700-0-200502-I/en
return opticalChannel < 0.018
? opticalChannel * 4.500
: 1.099 * pow(opticalChannel, inverseGamma) - 0.099;
}
// Transforms optical SDR colors to electrical SDR using the SMPTE 170M OETF.
vec3 sdrOetf(vec3 opticalColor) {
return uApplyOetf == 1
? vec3(
sdrOetfSingleChannel(opticalColor.r),
sdrOetfSingleChannel(opticalColor.g),
sdrOetfSingleChannel(opticalColor.b))
: opticalColor;
}
void main() {
vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoord);
vec3 linearInputColor = sdrEotf(inputColor.rgb);
vec4 transformedColors = uRgbMatrix * vec4(linearInputColor, 1);
gl_FragColor = vec4(sdrOetf(transformedColors.rgb), inputColor.a);
}
google3/third_party/java_src/android_libs/media/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_external_es2.glsl → google3/third_party/java_src/android_libs/media/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_oetf_es2.glsl
#version 100 #version 100
// Copyright 2021 The Android Open Source Project // Copyright 2022 The Android Open Source Project
// //
// Licensed under the Apache License, Version 2.0 (the "License"); // Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. // you may not use this file except in compliance with the License.
...@@ -13,18 +13,40 @@ ...@@ -13,18 +13,40 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
// ES 2 fragment shader that samples from an external texture with uTexSampler, // ES 2 fragment shader that:
// copying from this texture to the current output while applying a 4x4 RGB // 1. Samples from uTexSampler, copying from this texture to the current
// color matrix to change the pixel colors. // output.
// 2. Applies a 4x4 RGB color matrix to change the pixel colors.
// 3. Transforms the optical colors to electrical colors using the SMPTE
// 170M OETF.
#extension GL_OES_EGL_image_external : require
precision mediump float; precision mediump float;
uniform samplerExternalOES uTexSampler; uniform sampler2D uTexSampler;
uniform mat4 uRgbMatrix; uniform mat4 uRgbMatrix;
varying vec2 vTexSamplingCoord; varying vec2 vTexSamplingCoord;
const float inverseGamma = 0.4500;
// Transforms a single channel from optical to electrical SDR.
float sdrOetfSingleChannel(float opticalChannel) {
// Specification:
// https://www.itu.int/rec/R-REC-BT.1700-0-200502-I/en
return opticalChannel < 0.018
? opticalChannel * 4.500
: 1.099 * pow(opticalChannel, inverseGamma) - 0.099;
}
// Transforms optical SDR colors to electrical SDR using the SMPTE 170M OETF.
vec3 sdrOetf(vec3 opticalColor) {
return vec3(
sdrOetfSingleChannel(opticalColor.r),
sdrOetfSingleChannel(opticalColor.g),
sdrOetfSingleChannel(opticalColor.b));
}
void main() { void main() {
vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoord); vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoord);
gl_FragColor = uRgbMatrix * vec4(inputColor.rgb, 1); vec4 transformedColors = uRgbMatrix * vec4(inputColor.rgb, 1);
gl_FragColor.a = inputColor.a;
gl_FragColor = vec4(sdrOetf(transformedColors.rgb), inputColor.a);
} }
google3/third_party/java_src/android_libs/media/libraries/effect/src/main/java/androidx/media3/effect/MatrixTextureProcessor.java
...@@ -49,6 +49,7 @@ import java.util.List; ...@@ -49,6 +49,7 @@ import java.util.List;
* *
* <p>Can copy frames from an external texture and apply color transformations for HDR if needed. * <p>Can copy frames from an external texture and apply color transformations for HDR if needed.
*/ */
// TODO(b/241902517): Fix Gamma references since intermediate color space is now linear.
@SuppressWarnings("FunctionalInterfaceClash") // b/228192298 @SuppressWarnings("FunctionalInterfaceClash") // b/228192298
/* package */ final class MatrixTextureProcessor extends SingleFrameGlTextureProcessor /* package */ final class MatrixTextureProcessor extends SingleFrameGlTextureProcessor
implements ExternalTextureProcessor { implements ExternalTextureProcessor {
...@@ -61,10 +62,12 @@ import java.util.List; ...@@ -61,10 +62,12 @@ import java.util.List;
"shaders/fragment_shader_transformation_es2.glsl"; "shaders/fragment_shader_transformation_es2.glsl";
private static final String FRAGMENT_SHADER_OETF_ES3_PATH = private static final String FRAGMENT_SHADER_OETF_ES3_PATH =
"shaders/fragment_shader_oetf_es3.glsl"; "shaders/fragment_shader_oetf_es3.glsl";
private static final String FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_PATH = private static final String FRAGMENT_SHADER_TRANSFORMATION_SDR_OETF_ES2_PATH =
"shaders/fragment_shader_transformation_external_es2.glsl"; "shaders/fragment_shader_transformation_sdr_oetf_es2.glsl";
private static final String FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_YUV_ES3_PATH = private static final String FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_YUV_ES3_PATH =
"shaders/fragment_shader_transformation_external_yuv_es3.glsl"; "shaders/fragment_shader_transformation_external_yuv_es3.glsl";
private static final String FRAGMENT_SHADER_TRANSFORMATION_SDR_EXTERNAL_PATH =
"shaders/fragment_shader_transformation_sdr_external_es2.glsl";
private static final ImmutableList<float[]> NDC_SQUARE = private static final ImmutableList<float[]> NDC_SQUARE =
ImmutableList.of( ImmutableList.of(
new float[] {-1, -1, 0, 1}, new float[] {-1, -1, 0, 1},
...@@ -150,6 +153,7 @@ import java.util.List; ...@@ -150,6 +153,7 @@ import java.util.List;
context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_TRANSFORMATION_PATH); context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_TRANSFORMATION_PATH);
// No transfer functions needed, because input and output are both optical colors. // No transfer functions needed, because input and output are both optical colors.
// TODO(b/241902517): Add transfer functions since existing color filters may change the colors.
return new MatrixTextureProcessor( return new MatrixTextureProcessor(
glProgram, glProgram,
ImmutableList.copyOf(matrixTransformations), ImmutableList.copyOf(matrixTransformations),
...@@ -192,10 +196,9 @@ import java.util.List; ...@@ -192,10 +196,9 @@ import java.util.List;
String fragmentShaderFilePath = String fragmentShaderFilePath =
useHdr useHdr
? FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_YUV_ES3_PATH ? FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_YUV_ES3_PATH
: FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_PATH; : FRAGMENT_SHADER_TRANSFORMATION_SDR_EXTERNAL_PATH;
GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath); GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
// TODO(b/241902517): Implement gamma transfer functions.
if (useHdr) { if (useHdr) {
// In HDR editing mode the decoder output is sampled in YUV. // In HDR editing mode the decoder output is sampled in YUV.
if (!GlUtil.isYuvTargetExtensionSupported()) { if (!GlUtil.isYuvTargetExtensionSupported()) {
...@@ -212,6 +215,8 @@ import java.util.List; ...@@ -212,6 +215,8 @@ import java.util.List;
checkArgument( checkArgument(
colorTransfer == C.COLOR_TRANSFER_HLG || colorTransfer == C.COLOR_TRANSFER_ST2084); colorTransfer == C.COLOR_TRANSFER_HLG || colorTransfer == C.COLOR_TRANSFER_ST2084);
glProgram.setIntUniform("uEotfColorTransfer", colorTransfer); glProgram.setIntUniform("uEotfColorTransfer", colorTransfer);
} else {
glProgram.setIntUniform("uApplyOetf", 0);
} }
return new MatrixTextureProcessor( return new MatrixTextureProcessor(
...@@ -250,10 +255,9 @@ import java.util.List; ...@@ -250,10 +255,9 @@ import java.util.List;
String vertexShaderFilePath = String vertexShaderFilePath =
useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH; useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH;
String fragmentShaderFilePath = String fragmentShaderFilePath =
useHdr ? FRAGMENT_SHADER_OETF_ES3_PATH : FRAGMENT_SHADER_TRANSFORMATION_PATH; useHdr ? FRAGMENT_SHADER_OETF_ES3_PATH : FRAGMENT_SHADER_TRANSFORMATION_SDR_OETF_ES2_PATH;
GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath); GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
// TODO(b/241902517): Implement gamma transfer functions.
if (useHdr) { if (useHdr) {
@C.ColorTransfer int colorTransfer = electricalColorInfo.colorTransfer; @C.ColorTransfer int colorTransfer = electricalColorInfo.colorTransfer;
checkArgument( checkArgument(
...@@ -275,7 +279,7 @@ import java.util.List; ...@@ -275,7 +279,7 @@ import java.util.List;
* #setTextureTransformMatrix(float[])} to provide the transformation matrix associated with the * #setTextureTransformMatrix(float[])} to provide the transformation matrix associated with the
* external texture. * external texture.
* *
* <p>Applies the OETF, {@code matrixTransformations}, {@code rgbMatrices}, then the EOTF, to * <p>Applies the EOTF, {@code matrixTransformations}, {@code rgbMatrices}, then the OETF, to
* convert from and to input and output electrical colors. * convert from and to input and output electrical colors.
* *
* @param context The {@link Context}. * @param context The {@link Context}.
...@@ -299,10 +303,9 @@ import java.util.List; ...@@ -299,10 +303,9 @@ import java.util.List;
String fragmentShaderFilePath = String fragmentShaderFilePath =
useHdr useHdr
? FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_YUV_ES3_PATH ? FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_YUV_ES3_PATH
: FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_PATH; : FRAGMENT_SHADER_TRANSFORMATION_SDR_EXTERNAL_PATH;
GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath); GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
// TODO(b/241902517): Implement gamma transfer functions.
if (useHdr) { if (useHdr) {
// In HDR editing mode the decoder output is sampled in YUV. // In HDR editing mode the decoder output is sampled in YUV.
if (!GlUtil.isYuvTargetExtensionSupported()) { if (!GlUtil.isYuvTargetExtensionSupported()) {
...@@ -317,6 +320,8 @@ import java.util.List; ...@@ -317,6 +320,8 @@ import java.util.List;
// No transfer functions needed, because the EOTF and OETF cancel out. // No transfer functions needed, because the EOTF and OETF cancel out.
glProgram.setIntUniform("uEotfColorTransfer", Format.NO_VALUE); glProgram.setIntUniform("uEotfColorTransfer", Format.NO_VALUE);
} else {
glProgram.setIntUniform("uApplyOetf", 1);
} }
return new MatrixTextureProcessor( return new MatrixTextureProcessor(
......
library/transformer/src/test/java/com/google/android/exoplayer2/transformer/MssimCalculatorTest.java
...@@ -34,7 +34,7 @@ public class MssimCalculatorTest { ...@@ -34,7 +34,7 @@ public class MssimCalculatorTest {
@Test @Test
public void calculateSsim_sameImage() throws Exception { public void calculateSsim_sameImage() throws Exception {
Bitmap bitmap = readBitmap("media/bitmap/sample_mp4_first_frame/original.png"); Bitmap bitmap = readBitmap("media/bitmap/sample_mp4_first_frame/linear_colors/original.png");
byte[] imageLuminosities = bitmapToLuminosityArray(bitmap); byte[] imageLuminosities = bitmapToLuminosityArray(bitmap);
// SSIM equals 1 if the two images match. // SSIM equals 1 if the two images match.
...@@ -46,10 +46,11 @@ public class MssimCalculatorTest { ...@@ -46,10 +46,11 @@ public class MssimCalculatorTest {
@Test @Test
public void calculateSsim_increasedBrightness() throws Exception { public void calculateSsim_increasedBrightness() throws Exception {
Bitmap refBitmap = readBitmap("media/bitmap/sample_mp4_first_frame/original.png"); Bitmap refBitmap = readBitmap("media/bitmap/sample_mp4_first_frame/linear_colors/original.png");
Bitmap distBitmap = readBitmap("media/bitmap/sample_mp4_first_frame/increase_brightness.png"); Bitmap distBitmap =
readBitmap("media/bitmap/sample_mp4_first_frame/linear_colors/increase_brightness.png");
// SSIM as calculated by ffmpeg: 0.634326 = 63% // SSIM as calculated by ffmpeg: 0.526821 = 52%
assertThat( assertThat(
(int) (int)
...@@ -59,14 +60,15 @@ public class MssimCalculatorTest { ...@@ -59,14 +60,15 @@ public class MssimCalculatorTest {
refBitmap.getWidth(), refBitmap.getWidth(),
refBitmap.getHeight()) refBitmap.getHeight())
* 100)) * 100))
.isEqualTo(63); .isEqualTo(52);
} }
@Test @Test
public void calculateSsim_withWindowSkipping_similarToWithout() throws Exception { public void calculateSsim_withWindowSkipping_similarToWithout() throws Exception {
Bitmap referenceBitmap = readBitmap("media/bitmap/sample_mp4_first_frame/original.png"); Bitmap referenceBitmap =
readBitmap("media/bitmap/sample_mp4_first_frame/linear_colors/original.png");
Bitmap distortedBitmap = Bitmap distortedBitmap =
readBitmap("media/bitmap/sample_mp4_first_frame/increase_brightness.png"); readBitmap("media/bitmap/sample_mp4_first_frame/linear_colors/increase_brightness.png");
byte[] referenceLuminosity = bitmapToLuminosityArray(referenceBitmap); byte[] referenceLuminosity = bitmapToLuminosityArray(referenceBitmap);
byte[] distortedLuminosity = bitmapToLuminosityArray(distortedBitmap); byte[] distortedLuminosity = bitmapToLuminosityArray(distortedBitmap);
......
testdata/src/test/assets/media/bitmap/README.md
...@@ -24,7 +24,7 @@ To generate new "expected" assets: ...@@ -24,7 +24,7 @@ To generate new "expected" assets:
```shell ```shell
adb pull \ adb pull \
/sdcard/Android/data/androidx.media3.effect.test/cache/drawFrame_rotate90_actual.png \ /sdcard/Android/data/androidx.media3.effect.test/cache/drawFrame_rotate90_actual.png \
third_party/java_src/android_libs/media/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/rotate90.png third_party/java_src/android_libs/media/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/rotate90.png
``` ```
<!-- copybara:strip_end --> <!-- copybara:strip_end -->
testdata/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/README.md 0 → 100644
All the files are in *SMPTE 170M colors* following the
[ITU Spec](https://www.itu.int/rec/R-REC-BT.1700-0-200502-I/en) per the
[Android MediaFormat documentation](https://developer.android.com/reference/android/media/MediaFormat#COLOR_TRANSFER_SDR_VIDEO).
testdata/src/test/assets/media/bitmap/sample_mp4_first_frame/linear_colors/README.md 0 → 100644
All the files are in *Linear RGB* per the
[Android MediaFormat documentation](https://developer.android.com/reference/android/media/MediaFormat#COLOR_TRANSFER_LINEAR).
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment