Skip to content

SDK / exoplayer

Go to a project
Commit ce766d76 by tofunmi Committed by Tianyi Feng
Options

Create dynamically created shaders for multiple overlays. 

PiperOrigin-RevId: 496379904
parent d2a3d8f6
Expand all
Inline Side-by-side
Showing with 215 additions and 143 deletions
library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/OverlayTextureProcessorPixelTest.java
...@@ -70,6 +70,10 @@ public class OverlayTextureProcessorPixelTest { ...@@ -70,6 +70,10 @@ public class OverlayTextureProcessorPixelTest {
"media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_text_default.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_text_default.png";
public static final String OVERLAY_TEXT_TRANSLATE = public static final String OVERLAY_TEXT_TRANSLATE =
"media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_text_translate.png"; "media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_text_translate.png";
public static final String OVERLAY_MULTIPLE =
"media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_multiple.png";
public static final String OVERLAY_OVERLAP =
"media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_overlap.png";
private final Context context = getApplicationContext(); private final Context context = getApplicationContext();
...@@ -103,7 +107,7 @@ public class OverlayTextureProcessorPixelTest { ...@@ -103,7 +107,7 @@ public class OverlayTextureProcessorPixelTest {
@Test @Test
public void drawFrame_noOverlay_leavesFrameUnchanged() throws Exception { public void drawFrame_noOverlay_leavesFrameUnchanged() throws Exception {
String testId = "drawFrame_noOverlays"; String testId = "drawFrame_noOverlay";
overlayTextureProcessor = overlayTextureProcessor =
new OverlayEffect(/* textureOverlays= */ ImmutableList.of()) new OverlayEffect(/* textureOverlays= */ ImmutableList.of())
.toGlTextureProcessor(context, /* useHdr= */ false); .toGlTextureProcessor(context, /* useHdr= */ false);
...@@ -281,6 +285,77 @@ public class OverlayTextureProcessorPixelTest { ...@@ -281,6 +285,77 @@ public class OverlayTextureProcessorPixelTest {
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
} }
@Test
public void drawFrame_multipleOverlays_blendsBothIntoFrame() throws Exception {
String testId = "drawFrame_multipleOverlays";
float[] translateMatrix1 = GlUtil.create4x4IdentityMatrix();
Matrix.translateM(translateMatrix1, /* mOffset= */ 0, /* x= */ 0.5f, /* y= */ 0.5f, /* z= */ 1);
SpannableString overlayText = new SpannableString(/* source= */ "Overlay 1");
overlayText.setSpan(
new ForegroundColorSpan(Color.GRAY),
/* start= */ 0,
/* end= */ 4,
Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
OverlaySettings overlaySettings1 =
new OverlaySettings.Builder().setMatrix(translateMatrix1).build();
TextOverlay textOverlay = TextOverlay.createStaticTextOverlay(overlayText, overlaySettings1);
Bitmap bitmap = readBitmap(OVERLAY_PNG_ASSET_PATH);
OverlaySettings overlaySettings2 = new OverlaySettings.Builder().setAlpha(0.5f).build();
BitmapOverlay bitmapOverlay = BitmapOverlay.createStaticBitmapOverlay(bitmap, overlaySettings2);
overlayTextureProcessor =
new OverlayEffect(ImmutableList.of(textOverlay, bitmapOverlay))
.toGlTextureProcessor(context, /* useHdr= */ false);
Size outputSize = overlayTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = readBitmap(OVERLAY_MULTIPLE);
overlayTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
createArgb8888BitmapFromCurrentGlFramebuffer(outputSize.getWidth(), outputSize.getHeight());
maybeSaveTestBitmapToCacheDirectory(testId, /* bitmapLabel= */ "actual", actualBitmap);
float averagePixelAbsoluteDifference =
getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
@Test
public void drawFrame_overlappingOverlays_blendsOnFifoOrder() throws Exception {
String testId = "drawFrame_overlappingOverlays";
SpannableString overlayText = new SpannableString(/* source= */ "Overlapping text");
overlayText.setSpan(
new ForegroundColorSpan(Color.WHITE),
/* start= */ 0,
/* end= */ overlayText.length(),
Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
float[] scaleTextMatrix = GlUtil.create4x4IdentityMatrix();
Matrix.scaleM(scaleTextMatrix, /* mOffset= */ 0, /* x= */ 0.5f, /* y= */ 0.5f, /* z= */ 1);
OverlaySettings overlaySettings1 =
new OverlaySettings.Builder().setMatrix(scaleTextMatrix).build();
TextOverlay textOverlay = TextOverlay.createStaticTextOverlay(overlayText, overlaySettings1);
Bitmap bitmap = readBitmap(OVERLAY_PNG_ASSET_PATH);
float[] scaleMatrix = GlUtil.create4x4IdentityMatrix();
Matrix.scaleM(scaleMatrix, /* mOffset= */ 0, /* x= */ 3, /* y= */ 3, /* z= */ 1);
OverlaySettings overlaySettings2 = new OverlaySettings.Builder().setMatrix(scaleMatrix).build();
BitmapOverlay bitmapOverlay = BitmapOverlay.createStaticBitmapOverlay(bitmap, overlaySettings2);
overlayTextureProcessor =
new OverlayEffect(ImmutableList.of(bitmapOverlay, textOverlay))
.toGlTextureProcessor(context, /* useHdr= */ false);
Size outputSize = overlayTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = readBitmap(OVERLAY_OVERLAP);
overlayTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap =
createArgb8888BitmapFromCurrentGlFramebuffer(outputSize.getWidth(), outputSize.getHeight());
maybeSaveTestBitmapToCacheDirectory(testId, /* bitmapLabel= */ "actual", actualBitmap);
float averagePixelAbsoluteDifference =
getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
private void setupOutputTexture(int outputWidth, int outputHeight) throws GlUtil.GlException { private void setupOutputTexture(int outputWidth, int outputHeight) throws GlUtil.GlException {
int outputTexId = int outputTexId =
GlUtil.createTexture( GlUtil.createTexture(
......
library/effect/src/main/assets/shaders/fragment_shader_overlay_es2.glsl deleted 100644 → 0
#version 100
// Copyright 2022 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ES 2 fragment shader that overlays a bitmap over a video frame.
precision mediump float;
// Texture containing an input video frame.
uniform sampler2D uVideoTexSampler0;
// Texture containing the overlay bitmap.
uniform sampler2D uOverlayTexSampler1;
// The alpha values for the texture.
uniform float uOverlayAlpha1;
varying vec2 vVideoTexSamplingCoord;
varying vec2 vOverlayTexSamplingCoord1;
// Manually implementing the CLAMP_TO_BORDER texture wrapping option
// (https://open.gl/textures) since it's not implemented until OpenGL ES 3.2.
vec4 getClampToBorderOverlayColor() {
if (vOverlayTexSamplingCoord1.x > 1.0 || vOverlayTexSamplingCoord1.x < 0.0
|| vOverlayTexSamplingCoord1.y > 1.0 || vOverlayTexSamplingCoord1.y < 0.0) {
return vec4(0.0, 0.0, 0.0, 0.0);
} else {
vec4 overlayColor = vec4(
texture2D(uOverlayTexSampler1, vOverlayTexSamplingCoord1));
overlayColor.a = uOverlayAlpha1 * overlayColor.a;
return overlayColor;
}
}
float getMixAlpha(float videoAlpha, float overlayAlpha) {
if (videoAlpha == 0.0) {
return 1.0;
} else {
return clamp(overlayAlpha/videoAlpha, 0.0, 1.0);
}
}
void main() {
vec4 videoColor = vec4(texture2D(uVideoTexSampler0, vVideoTexSamplingCoord));
vec4 overlayColor = getClampToBorderOverlayColor();
// Blend the video decoder output and the overlay bitmap.
gl_FragColor = mix(
videoColor, overlayColor, getMixAlpha(videoColor.a, overlayColor.a));
}
library/effect/src/main/assets/shaders/vertex_shader_overlay_es2.glsl deleted 100644 → 0
#version 100
// Copyright 2022 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ES 2 vertex shader that leaves the frame coordinates unchanged
// and applies matrix transformations to the texture coordinates.
uniform mat4 uAspectRatioMatrix;
uniform mat4 uOverlayMatrix;
attribute vec4 aFramePosition;
varying vec2 vVideoTexSamplingCoord;
varying vec2 vOverlayTexSamplingCoord1;
vec2 getTexSamplingCoord(vec2 ndcPosition) {
return vec2(ndcPosition.x * 0.5 + 0.5, ndcPosition.y * 0.5 + 0.5);
}
void main() {
gl_Position = aFramePosition;
vec4 aOverlayPosition = uAspectRatioMatrix * uOverlayMatrix * aFramePosition;
vOverlayTexSamplingCoord1 = getTexSamplingCoord(aOverlayPosition.xy);
vVideoTexSamplingCoord = getTexSamplingCoord(aFramePosition.xy);
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment