Skip to content

SDK / exoplayer

Go to a project
Commit 23b0610a by hschlueter Committed by microkatz
Options

Move program initialization to texture processor constructor. 

Once the more advanced GlTextureProcessor interface exists,
it will be possible to change the output size of a GlTextureProcessor
between frames. To keep the re-configuration based on the frame sizes
minimal, things indepedent of the frame size, such as the GlProgram,
can be initialized in the constructor.

PiperOrigin-RevId: 451997584
(cherry picked from commit 54d44d38)
parent f7fc04dc
Expand all
Inline Side-by-side
Showing with 263 additions and 249 deletions
demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/BitmapOverlayProcessor.java
...@@ -35,7 +35,6 @@ import com.google.android.exoplayer2.util.GlProgram; ...@@ -35,7 +35,6 @@ import com.google.android.exoplayer2.util.GlProgram;
import com.google.android.exoplayer2.util.GlUtil; import com.google.android.exoplayer2.util.GlUtil;
import java.io.IOException; import java.io.IOException;
import java.util.Locale; import java.util.Locale;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** /**
* A {@link SingleFrameGlTextureProcessor} that overlays a bitmap with a logo and timer on each * A {@link SingleFrameGlTextureProcessor} that overlays a bitmap with a logo and timer on each
...@@ -57,16 +56,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -57,16 +56,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final Paint paint; private final Paint paint;
private final Bitmap overlayBitmap; private final Bitmap overlayBitmap;
private final Bitmap logoBitmap;
private final Canvas overlayCanvas; private final Canvas overlayCanvas;
private final GlProgram glProgram;
private float bitmapScaleX; private float bitmapScaleX;
private float bitmapScaleY; private float bitmapScaleY;
private int bitmapTexId; private int bitmapTexId;
private @MonotonicNonNull Size outputSize;
private @MonotonicNonNull Bitmap logoBitmap;
private @MonotonicNonNull GlProgram glProgram;
public BitmapOverlayProcessor() { /**
* Creates a new instance.
*
* @throws IOException If a problem occurs while reading shader files.
*/
public BitmapOverlayProcessor(Context context) throws IOException {
paint = new Paint(); paint = new Paint();
paint.setTextSize(64); paint.setTextSize(64);
paint.setAntiAlias(true); paint.setAntiAlias(true);
...@@ -75,19 +78,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -75,19 +78,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
overlayBitmap = overlayBitmap =
Bitmap.createBitmap(BITMAP_WIDTH_HEIGHT, BITMAP_WIDTH_HEIGHT, Bitmap.Config.ARGB_8888); Bitmap.createBitmap(BITMAP_WIDTH_HEIGHT, BITMAP_WIDTH_HEIGHT, Bitmap.Config.ARGB_8888);
overlayCanvas = new Canvas(overlayBitmap); overlayCanvas = new Canvas(overlayBitmap);
}
@Override
public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight)
throws IOException {
if (inputWidth > inputHeight) {
bitmapScaleX = inputWidth / (float) inputHeight;
bitmapScaleY = 1f;
} else {
bitmapScaleX = 1f;
bitmapScaleY = inputHeight / (float) inputWidth;
}
outputSize = new Size(inputWidth, inputHeight);
try { try {
logoBitmap = logoBitmap =
...@@ -106,19 +96,27 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -106,19 +96,27 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
"aFramePosition", "aFramePosition",
GlUtil.getNormalizedCoordinateBounds(), GlUtil.getNormalizedCoordinateBounds(),
GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
glProgram.setSamplerTexIdUniform("uTexSampler0", inputTexId, /* texUnitIndex= */ 0);
glProgram.setSamplerTexIdUniform("uTexSampler1", bitmapTexId, /* texUnitIndex= */ 1); glProgram.setSamplerTexIdUniform("uTexSampler1", bitmapTexId, /* texUnitIndex= */ 1);
glProgram.setFloatUniform("uScaleX", bitmapScaleX);
glProgram.setFloatUniform("uScaleY", bitmapScaleY);
} }
@Override @Override
public Size getOutputSize() { public Size configure(int inputWidth, int inputHeight) {
return checkStateNotNull(outputSize); if (inputWidth > inputHeight) {
bitmapScaleX = inputWidth / (float) inputHeight;
bitmapScaleY = 1f;
} else {
bitmapScaleX = 1f;
bitmapScaleY = inputHeight / (float) inputWidth;
}
glProgram.setFloatUniform("uScaleX", bitmapScaleX);
glProgram.setFloatUniform("uScaleY", bitmapScaleY);
return new Size(inputWidth, inputHeight);
} }
@Override @Override
public void drawFrame(long presentationTimeUs) throws FrameProcessingException { public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
try { try {
checkStateNotNull(glProgram).use(); checkStateNotNull(glProgram).use();
...@@ -137,6 +135,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -137,6 +135,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
flipBitmapVertically(overlayBitmap)); flipBitmapVertically(overlayBitmap));
GlUtil.checkGlError(); GlUtil.checkGlError();
glProgram.setSamplerTexIdUniform("uTexSampler0", inputTexId, /* texUnitIndex= */ 0);
glProgram.bindAttributesAndUniforms(); glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad. // The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
......
demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/PeriodicVignetteProcessor.java
...@@ -16,7 +16,6 @@ ...@@ -16,7 +16,6 @@
package com.google.android.exoplayer2.transformerdemo; package com.google.android.exoplayer2.transformerdemo;
import static com.google.android.exoplayer2.util.Assertions.checkArgument; import static com.google.android.exoplayer2.util.Assertions.checkArgument;
import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
import android.content.Context; import android.content.Context;
import android.opengl.GLES20; import android.opengl.GLES20;
...@@ -26,7 +25,6 @@ import com.google.android.exoplayer2.transformer.SingleFrameGlTextureProcessor; ...@@ -26,7 +25,6 @@ import com.google.android.exoplayer2.transformer.SingleFrameGlTextureProcessor;
import com.google.android.exoplayer2.util.GlProgram; import com.google.android.exoplayer2.util.GlProgram;
import com.google.android.exoplayer2.util.GlUtil; import com.google.android.exoplayer2.util.GlUtil;
import java.io.IOException; import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** /**
* A {@link SingleFrameGlTextureProcessor} that periodically dims the frames such that pixels are * A {@link SingleFrameGlTextureProcessor} that periodically dims the frames such that pixels are
...@@ -41,14 +39,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -41,14 +39,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private static final String FRAGMENT_SHADER_PATH = "fragment_shader_vignette_es2.glsl"; private static final String FRAGMENT_SHADER_PATH = "fragment_shader_vignette_es2.glsl";
private static final float DIMMING_PERIOD_US = 5_600_000f; private static final float DIMMING_PERIOD_US = 5_600_000f;
private float centerX; private final GlProgram glProgram;
private float centerY; private final float minInnerRadius;
private float minInnerRadius; private final float deltaInnerRadius;
private float deltaInnerRadius;
private float outerRadius;
private @MonotonicNonNull Size outputSize;
private @MonotonicNonNull GlProgram glProgram;
/** /**
* Creates a new instance. * Creates a new instance.
...@@ -61,29 +54,27 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -61,29 +54,27 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* *
* <p>The parameters are given in normalized texture coordinates from 0 to 1. * <p>The parameters are given in normalized texture coordinates from 0 to 1.
* *
* @param context The {@link Context}.
* @param centerX The x-coordinate of the center of the effect. * @param centerX The x-coordinate of the center of the effect.
* @param centerY The y-coordinate of the center of the effect. * @param centerY The y-coordinate of the center of the effect.
* @param minInnerRadius The lower bound of the radius that is unaffected by the effect. * @param minInnerRadius The lower bound of the radius that is unaffected by the effect.
* @param maxInnerRadius The upper bound of the radius that is unaffected by the effect. * @param maxInnerRadius The upper bound of the radius that is unaffected by the effect.
* @param outerRadius The radius after which all pixels are black. * @param outerRadius The radius after which all pixels are black.
* @throws IOException If a problem occurs while reading shader files.
*/ */
public PeriodicVignetteProcessor( public PeriodicVignetteProcessor(
float centerX, float centerY, float minInnerRadius, float maxInnerRadius, float outerRadius) { Context context,
float centerX,
float centerY,
float minInnerRadius,
float maxInnerRadius,
float outerRadius)
throws IOException {
checkArgument(minInnerRadius <= maxInnerRadius); checkArgument(minInnerRadius <= maxInnerRadius);
checkArgument(maxInnerRadius <= outerRadius); checkArgument(maxInnerRadius <= outerRadius);
this.centerX = centerX;
this.centerY = centerY;
this.minInnerRadius = minInnerRadius; this.minInnerRadius = minInnerRadius;
this.deltaInnerRadius = maxInnerRadius - minInnerRadius; this.deltaInnerRadius = maxInnerRadius - minInnerRadius;
this.outerRadius = outerRadius;
}
@Override
public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight)
throws IOException {
outputSize = new Size(inputWidth, inputHeight);
glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH); glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH);
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
glProgram.setFloatsUniform("uCenter", new float[] {centerX, centerY}); glProgram.setFloatsUniform("uCenter", new float[] {centerX, centerY});
glProgram.setFloatsUniform("uOuterRadius", new float[] {outerRadius}); glProgram.setFloatsUniform("uOuterRadius", new float[] {outerRadius});
// Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y. // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y.
...@@ -94,14 +85,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -94,14 +85,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
@Override @Override
public Size getOutputSize() { public Size configure(int inputWidth, int inputHeight) {
return checkStateNotNull(outputSize); return new Size(inputWidth, inputHeight);
} }
@Override @Override
public void drawFrame(long presentationTimeUs) throws FrameProcessingException { public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
try { try {
checkStateNotNull(glProgram).use(); glProgram.use();
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
double theta = presentationTimeUs * 2 * Math.PI / DIMMING_PERIOD_US; double theta = presentationTimeUs * 2 * Math.PI / DIMMING_PERIOD_US;
float innerRadius = float innerRadius =
minInnerRadius + deltaInnerRadius * (0.5f - 0.5f * (float) Math.cos(theta)); minInnerRadius + deltaInnerRadius * (0.5f - 0.5f * (float) Math.cos(theta));
......
demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java
...@@ -19,6 +19,7 @@ import static android.Manifest.permission.READ_EXTERNAL_STORAGE; ...@@ -19,6 +19,7 @@ import static android.Manifest.permission.READ_EXTERNAL_STORAGE;
import static com.google.android.exoplayer2.util.Assertions.checkNotNull; import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import android.app.Activity; import android.app.Activity;
import android.content.Context;
import android.content.Intent; import android.content.Intent;
import android.content.pm.PackageManager; import android.content.pm.PackageManager;
import android.net.Uri; import android.net.Uri;
...@@ -275,12 +276,13 @@ public final class TransformerActivity extends AppCompatActivity { ...@@ -275,12 +276,13 @@ public final class TransformerActivity extends AppCompatActivity {
Class<?> clazz = Class<?> clazz =
Class.forName("com.google.android.exoplayer2.transformerdemo.MediaPipeProcessor"); Class.forName("com.google.android.exoplayer2.transformerdemo.MediaPipeProcessor");
Constructor<?> constructor = Constructor<?> constructor =
clazz.getConstructor(String.class, String.class, String.class); clazz.getConstructor(Context.class, String.class, String.class, String.class);
effects.add( effects.add(
() -> { (Context context) -> {
try { try {
return (SingleFrameGlTextureProcessor) return (SingleFrameGlTextureProcessor)
constructor.newInstance( constructor.newInstance(
context,
/* graphName= */ "edge_detector_mediapipe_graph.binarypb", /* graphName= */ "edge_detector_mediapipe_graph.binarypb",
/* inputStreamName= */ "input_video", /* inputStreamName= */ "input_video",
/* outputStreamName= */ "output_video"); /* outputStreamName= */ "output_video");
...@@ -295,8 +297,9 @@ public final class TransformerActivity extends AppCompatActivity { ...@@ -295,8 +297,9 @@ public final class TransformerActivity extends AppCompatActivity {
} }
if (selectedEffects[2]) { if (selectedEffects[2]) {
effects.add( effects.add(
() -> (Context context) ->
new PeriodicVignetteProcessor( new PeriodicVignetteProcessor(
context,
bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_X), bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_X),
bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_Y), bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_Y),
/* minInnerRadius= */ bundle.getFloat( /* minInnerRadius= */ bundle.getFloat(
......
demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java
...@@ -63,49 +63,36 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -63,49 +63,36 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private static final String COPY_VERTEX_SHADER_NAME = "vertex_shader_copy_es2.glsl"; private static final String COPY_VERTEX_SHADER_NAME = "vertex_shader_copy_es2.glsl";
private static final String COPY_FRAGMENT_SHADER_NAME = "shaders/fragment_shader_copy_es2.glsl"; private static final String COPY_FRAGMENT_SHADER_NAME = "shaders/fragment_shader_copy_es2.glsl";
private final String graphName;
private final String inputStreamName;
private final String outputStreamName;
private final ConditionVariable frameProcessorConditionVariable; private final ConditionVariable frameProcessorConditionVariable;
private final FrameProcessor frameProcessor;
private final GlProgram glProgram;
private @MonotonicNonNull FrameProcessor frameProcessor;
private int inputWidth; private int inputWidth;
private int inputHeight; private int inputHeight;
private int inputTexId;
private @MonotonicNonNull GlProgram glProgram;
private @MonotonicNonNull TextureFrame outputFrame; private @MonotonicNonNull TextureFrame outputFrame;
private @MonotonicNonNull RuntimeException frameProcessorPendingError; private @MonotonicNonNull RuntimeException frameProcessorPendingError;
/** /**
* Creates a new texture processor that wraps a MediaPipe graph. * Creates a new texture processor that wraps a MediaPipe graph.
* *
* @param context The {@link Context}.
* @param graphName Name of a MediaPipe graph asset to load. * @param graphName Name of a MediaPipe graph asset to load.
* @param inputStreamName Name of the input video stream in the graph. * @param inputStreamName Name of the input video stream in the graph.
* @param outputStreamName Name of the input video stream in the graph. * @param outputStreamName Name of the input video stream in the graph.
* @throws IOException If a problem occurs while reading shader files or initializing MediaPipe
* resources.
*/ */
public MediaPipeProcessor(String graphName, String inputStreamName, String outputStreamName) { public MediaPipeProcessor(
checkState(LOADER.isAvailable()); Context context, String graphName, String inputStreamName, String outputStreamName)
this.graphName = graphName;
this.inputStreamName = inputStreamName;
this.outputStreamName = outputStreamName;
frameProcessorConditionVariable = new ConditionVariable();
}
@Override
public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight)
throws IOException { throws IOException {
this.inputTexId = inputTexId; checkState(LOADER.isAvailable());
this.inputWidth = inputWidth;
this.inputHeight = inputHeight;
glProgram = new GlProgram(context, COPY_VERTEX_SHADER_NAME, COPY_FRAGMENT_SHADER_NAME);
frameProcessorConditionVariable = new ConditionVariable();
AndroidAssetUtil.initializeNativeAssetManager(context); AndroidAssetUtil.initializeNativeAssetManager(context);
EglManager eglManager = new EglManager(EGL14.eglGetCurrentContext()); EglManager eglManager = new EglManager(EGL14.eglGetCurrentContext());
frameProcessor = frameProcessor =
new FrameProcessor( new FrameProcessor(
context, eglManager.getNativeContext(), graphName, inputStreamName, outputStreamName); context, eglManager.getNativeContext(), graphName, inputStreamName, outputStreamName);
// Unblock drawFrame when there is an output frame or an error. // Unblock drawFrame when there is an output frame or an error.
frameProcessor.setConsumer( frameProcessor.setConsumer(
frame -> { frame -> {
...@@ -117,15 +104,18 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -117,15 +104,18 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
frameProcessorPendingError = error; frameProcessorPendingError = error;
frameProcessorConditionVariable.open(); frameProcessorConditionVariable.open();
}); });
glProgram = new GlProgram(context, COPY_VERTEX_SHADER_NAME, COPY_FRAGMENT_SHADER_NAME);
} }
@Override @Override
public Size getOutputSize() { public Size configure(int inputWidth, int inputHeight) {
this.inputWidth = inputWidth;
this.inputHeight = inputHeight;
return new Size(inputWidth, inputHeight); return new Size(inputWidth, inputHeight);
} }
@Override @Override
public void drawFrame(long presentationTimeUs) throws FrameProcessingException { public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
frameProcessorConditionVariable.close(); frameProcessorConditionVariable.close();
// Pass the input frame to MediaPipe. // Pass the input frame to MediaPipe.
......
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameProcessorChainTest.java
...@@ -122,7 +122,7 @@ public final class FrameProcessorChainTest { ...@@ -122,7 +122,7 @@ public final class FrameProcessorChainTest {
throws FrameProcessingException { throws FrameProcessingException {
ImmutableList.Builder<GlEffect> effects = new ImmutableList.Builder<>(); ImmutableList.Builder<GlEffect> effects = new ImmutableList.Builder<>();
for (Size element : textureProcessorOutputSizes) { for (Size element : textureProcessorOutputSizes) {
effects.add(() -> new FakeTextureProcessor(element)); effects.add((Context context) -> new FakeTextureProcessor(element));
} }
return FrameProcessorChain.create( return FrameProcessorChain.create(
getApplicationContext(), getApplicationContext(),
...@@ -144,15 +144,12 @@ public final class FrameProcessorChainTest { ...@@ -144,15 +144,12 @@ public final class FrameProcessorChainTest {
} }
@Override @Override
public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight) {} public Size configure(int inputWidth, int inputHeight) {
@Override
public Size getOutputSize() {
return outputSize; return outputSize;
} }
@Override @Override
public void drawFrame(long presentationTimeNs) {} public void drawFrame(int inputTexId, long presentationTimeNs) {}
@Override @Override
public void release() {} public void release() {}
......
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/MatrixTransformationProcessorPixelTest.java
...@@ -19,6 +19,7 @@ import static androidx.test.core.app.ApplicationProvider.getApplicationContext; ...@@ -19,6 +19,7 @@ import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
import static com.google.android.exoplayer2.transformer.BitmapTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE; import static com.google.android.exoplayer2.transformer.BitmapTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE;
import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertThat;
import android.content.Context;
import android.graphics.Bitmap; import android.graphics.Bitmap;
import android.graphics.Matrix; import android.graphics.Matrix;
import android.opengl.EGLContext; import android.opengl.EGLContext;
...@@ -56,9 +57,10 @@ public final class MatrixTransformationProcessorPixelTest { ...@@ -56,9 +57,10 @@ public final class MatrixTransformationProcessorPixelTest {
GlUtil.glAssertionsEnabled = true; GlUtil.glAssertionsEnabled = true;
} }
private final Context context = getApplicationContext();
private final EGLDisplay eglDisplay = GlUtil.createEglDisplay(); private final EGLDisplay eglDisplay = GlUtil.createEglDisplay();
private final EGLContext eglContext = GlUtil.createEglContext(eglDisplay); private final EGLContext eglContext = GlUtil.createEglContext(eglDisplay);
private @MonotonicNonNull SingleFrameGlTextureProcessor matrixTransformationProcessor; private @MonotonicNonNull SingleFrameGlTextureProcessor matrixTransformationFrameProcessor;
private int inputTexId; private int inputTexId;
private int outputTexId; private int outputTexId;
private int width; private int width;
...@@ -80,8 +82,8 @@ public final class MatrixTransformationProcessorPixelTest { ...@@ -80,8 +82,8 @@ public final class MatrixTransformationProcessorPixelTest {
@After @After
public void release() { public void release() {
if (matrixTransformationProcessor != null) { if (matrixTransformationFrameProcessor != null) {
matrixTransformationProcessor.release(); matrixTransformationFrameProcessor.release();
} }
GlUtil.destroyEglContext(eglDisplay, eglContext); GlUtil.destroyEglContext(eglDisplay, eglContext);
} }
...@@ -90,12 +92,12 @@ public final class MatrixTransformationProcessorPixelTest { ...@@ -90,12 +92,12 @@ public final class MatrixTransformationProcessorPixelTest {
public void drawFrame_noEdits_producesExpectedOutput() throws Exception { public void drawFrame_noEdits_producesExpectedOutput() throws Exception {
String testId = "drawFrame_noEdits"; String testId = "drawFrame_noEdits";
Matrix identityMatrix = new Matrix(); Matrix identityMatrix = new Matrix();
matrixTransformationProcessor = matrixTransformationFrameProcessor =
new MatrixTransformationProcessor((long presentationTimeUs) -> identityMatrix); new MatrixTransformationProcessor(context, (long presentationTimeUs) -> identityMatrix);
matrixTransformationProcessor.initialize(getApplicationContext(), inputTexId, width, height); matrixTransformationFrameProcessor.configure(width, height);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH);
matrixTransformationProcessor.drawFrame(/* presentationTimeUs= */ 0); matrixTransformationFrameProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap = Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height); BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height);
...@@ -113,12 +115,13 @@ public final class MatrixTransformationProcessorPixelTest { ...@@ -113,12 +115,13 @@ public final class MatrixTransformationProcessorPixelTest {
String testId = "drawFrame_translateRight"; String testId = "drawFrame_translateRight";
Matrix translateRightMatrix = new Matrix(); Matrix translateRightMatrix = new Matrix();
translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0); translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0);
matrixTransformationProcessor = matrixTransformationFrameProcessor =
new MatrixTransformationProcessor((long presentationTimeUs) -> translateRightMatrix); new MatrixTransformationProcessor(
matrixTransformationProcessor.initialize(getApplicationContext(), inputTexId, width, height); context, /* matrixTransformation= */ (long presentationTimeUs) -> translateRightMatrix);
matrixTransformationFrameProcessor.configure(width, height);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_PNG_ASSET_PATH); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_PNG_ASSET_PATH);
matrixTransformationProcessor.drawFrame(/* presentationTimeUs= */ 0); matrixTransformationFrameProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap = Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height); BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height);
...@@ -136,12 +139,13 @@ public final class MatrixTransformationProcessorPixelTest { ...@@ -136,12 +139,13 @@ public final class MatrixTransformationProcessorPixelTest {
String testId = "drawFrame_scaleNarrow"; String testId = "drawFrame_scaleNarrow";
Matrix scaleNarrowMatrix = new Matrix(); Matrix scaleNarrowMatrix = new Matrix();
scaleNarrowMatrix.postScale(.5f, 1.2f); scaleNarrowMatrix.postScale(.5f, 1.2f);
matrixTransformationProcessor = matrixTransformationFrameProcessor =
new MatrixTransformationProcessor((long presentationTimeUs) -> scaleNarrowMatrix); new MatrixTransformationProcessor(
matrixTransformationProcessor.initialize(getApplicationContext(), inputTexId, width, height); context, /* matrixTransformation= */ (long presentationTimeUs) -> scaleNarrowMatrix);
matrixTransformationFrameProcessor.configure(width, height);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(SCALE_NARROW_PNG_ASSET_PATH); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(SCALE_NARROW_PNG_ASSET_PATH);
matrixTransformationProcessor.drawFrame(/* presentationTimeUs= */ 0); matrixTransformationFrameProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap = Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height); BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height);
...@@ -159,12 +163,13 @@ public final class MatrixTransformationProcessorPixelTest { ...@@ -159,12 +163,13 @@ public final class MatrixTransformationProcessorPixelTest {
String testId = "drawFrame_rotate90"; String testId = "drawFrame_rotate90";
Matrix rotate90Matrix = new Matrix(); Matrix rotate90Matrix = new Matrix();
rotate90Matrix.postRotate(/* degrees= */ 90); rotate90Matrix.postRotate(/* degrees= */ 90);
matrixTransformationProcessor = matrixTransformationFrameProcessor =
new MatrixTransformationProcessor((long presentationTimeUs) -> rotate90Matrix); new MatrixTransformationProcessor(
matrixTransformationProcessor.initialize(getApplicationContext(), inputTexId, width, height); context, /* matrixTransformation= */ (long presentationTimeUs) -> rotate90Matrix);
matrixTransformationFrameProcessor.configure(width, height);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_90_PNG_ASSET_PATH); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_90_PNG_ASSET_PATH);
matrixTransformationProcessor.drawFrame(/* presentationTimeUs= */ 0); matrixTransformationFrameProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0);
Bitmap actualBitmap = Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height); BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height);
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExternalTextureProcessor.java
...@@ -24,7 +24,6 @@ import android.util.Size; ...@@ -24,7 +24,6 @@ import android.util.Size;
import com.google.android.exoplayer2.util.GlProgram; import com.google.android.exoplayer2.util.GlProgram;
import com.google.android.exoplayer2.util.GlUtil; import com.google.android.exoplayer2.util.GlUtil;
import java.io.IOException; import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** Copies frames from an external texture and applies color transformations for HDR if needed. */ /** Copies frames from an external texture and applies color transformations for HDR if needed. */
/* package */ class ExternalTextureProcessor implements SingleFrameGlTextureProcessor { /* package */ class ExternalTextureProcessor implements SingleFrameGlTextureProcessor {
...@@ -49,22 +48,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -49,22 +48,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
1.683f, -0.652f, 0.0f, 1.683f, -0.652f, 0.0f,
}; };
private final boolean enableExperimentalHdrEditing; private final GlProgram glProgram;
private @MonotonicNonNull Size size; /**
private @MonotonicNonNull GlProgram glProgram; * Creates a new instance.
*
public ExternalTextureProcessor(boolean enableExperimentalHdrEditing) { * @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal.
this.enableExperimentalHdrEditing = enableExperimentalHdrEditing; * @throws IOException If a problem occurs while reading shader files.
} */
public ExternalTextureProcessor(Context context, boolean enableExperimentalHdrEditing)
@Override
public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight)
throws IOException { throws IOException {
checkArgument(inputWidth > 0, "inputWidth must be positive");
checkArgument(inputHeight > 0, "inputHeight must be positive");
size = new Size(inputWidth, inputHeight);
String vertexShaderFilePath = String vertexShaderFilePath =
enableExperimentalHdrEditing enableExperimentalHdrEditing
? VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH ? VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH
...@@ -74,7 +67,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -74,7 +67,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH
: FRAGMENT_SHADER_COPY_EXTERNAL_PATH; : FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath); glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
// Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y. // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y.
glProgram.setBufferAttribute( glProgram.setBufferAttribute(
"aFramePosition", "aFramePosition",
...@@ -87,8 +79,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -87,8 +79,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
@Override @Override
public Size getOutputSize() { public Size configure(int inputWidth, int inputHeight) {
return checkStateNotNull(size); checkArgument(inputWidth > 0, "inputWidth must be positive");
checkArgument(inputHeight > 0, "inputHeight must be positive");
return new Size(inputWidth, inputHeight);
} }
/** /**
...@@ -104,10 +99,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -104,10 +99,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
@Override @Override
public void drawFrame(long presentationTimeUs) throws FrameProcessingException { public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
checkStateNotNull(glProgram); checkStateNotNull(glProgram);
try { try {
glProgram.use(); glProgram.use();
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
glProgram.bindAttributesAndUniforms(); glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad. // The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameProcessorChain.java
...@@ -171,22 +171,21 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -171,22 +171,21 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
ExternalTextureProcessor externalTextureProcessor = ExternalTextureProcessor externalTextureProcessor =
new ExternalTextureProcessor(enableExperimentalHdrEditing); new ExternalTextureProcessor(context, enableExperimentalHdrEditing);
ImmutableList<SingleFrameGlTextureProcessor> textureProcessors = ImmutableList<SingleFrameGlTextureProcessor> textureProcessors =
getTextureProcessors(externalTextureProcessor, pixelWidthHeightRatio, effects); getTextureProcessors(context, externalTextureProcessor, pixelWidthHeightRatio, effects);
// Initialize texture processors. // Initialize texture processors.
int inputExternalTexId = GlUtil.createExternalTexture(); int inputExternalTexId = GlUtil.createExternalTexture();
externalTextureProcessor.initialize(context, inputExternalTexId, inputWidth, inputHeight); Size outputSize = externalTextureProcessor.configure(inputWidth, inputHeight);
ImmutableList.Builder<TextureInfo> intermediateTextures = new ImmutableList.Builder<>();
int[] framebuffers = new int[textureProcessors.size() - 1];
Size inputSize = externalTextureProcessor.getOutputSize();
for (int i = 1; i < textureProcessors.size(); i++) { for (int i = 1; i < textureProcessors.size(); i++) {
int inputTexId = GlUtil.createTexture(inputSize.getWidth(), inputSize.getHeight()); int texId = GlUtil.createTexture(outputSize.getWidth(), outputSize.getHeight());
framebuffers[i - 1] = GlUtil.createFboForTexture(inputTexId); int fboId = GlUtil.createFboForTexture(texId);
intermediateTextures.add(
new TextureInfo(texId, fboId, outputSize.getWidth(), outputSize.getHeight()));
SingleFrameGlTextureProcessor textureProcessor = textureProcessors.get(i); SingleFrameGlTextureProcessor textureProcessor = textureProcessors.get(i);
textureProcessor.initialize(context, inputTexId, inputSize.getWidth(), inputSize.getHeight()); outputSize = textureProcessor.configure(outputSize.getWidth(), outputSize.getHeight());
inputSize = textureProcessor.getOutputSize();
} }
return new FrameProcessorChain( return new FrameProcessorChain(
eglDisplay, eglDisplay,
...@@ -194,16 +193,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -194,16 +193,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
singleThreadExecutorService, singleThreadExecutorService,
inputExternalTexId, inputExternalTexId,
streamOffsetUs, streamOffsetUs,
framebuffers, intermediateTextures.build(),
textureProcessors, textureProcessors,
outputSize,
listener, listener,
enableExperimentalHdrEditing); enableExperimentalHdrEditing);
} }
private static ImmutableList<SingleFrameGlTextureProcessor> getTextureProcessors( private static ImmutableList<SingleFrameGlTextureProcessor> getTextureProcessors(
Context context,
ExternalTextureProcessor externalTextureProcessor, ExternalTextureProcessor externalTextureProcessor,
float pixelWidthHeightRatio, float pixelWidthHeightRatio,
List<GlEffect> effects) { List<GlEffect> effects)
throws IOException {
ImmutableList.Builder<SingleFrameGlTextureProcessor> textureProcessors = ImmutableList.Builder<SingleFrameGlTextureProcessor> textureProcessors =
new ImmutableList.Builder<SingleFrameGlTextureProcessor>().add(externalTextureProcessor); new ImmutableList.Builder<SingleFrameGlTextureProcessor>().add(externalTextureProcessor);
...@@ -233,15 +235,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -233,15 +235,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
ImmutableList<GlMatrixTransformation> matrixTransformations = ImmutableList<GlMatrixTransformation> matrixTransformations =
matrixTransformationListBuilder.build(); matrixTransformationListBuilder.build();
if (!matrixTransformations.isEmpty()) { if (!matrixTransformations.isEmpty()) {
textureProcessors.add(new MatrixTransformationProcessor(matrixTransformations)); textureProcessors.add(new MatrixTransformationProcessor(context, matrixTransformations));
matrixTransformationListBuilder = new ImmutableList.Builder<>(); matrixTransformationListBuilder = new ImmutableList.Builder<>();
} }
textureProcessors.add(effect.toGlTextureProcessor()); textureProcessors.add(effect.toGlTextureProcessor(context));
} }
ImmutableList<GlMatrixTransformation> matrixTransformations = ImmutableList<GlMatrixTransformation> matrixTransformations =
matrixTransformationListBuilder.build(); matrixTransformationListBuilder.build();
if (!matrixTransformations.isEmpty()) { if (!matrixTransformations.isEmpty()) {
textureProcessors.add(new MatrixTransformationProcessor(matrixTransformations)); textureProcessors.add(new MatrixTransformationProcessor(context, matrixTransformations));
} }
return textureProcessors.build(); return textureProcessors.build();
...@@ -265,11 +267,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -265,11 +267,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final ConcurrentLinkedQueue<Future<?>> futures; private final ConcurrentLinkedQueue<Future<?>> futures;
/** Number of frames {@linkplain #registerInputFrame() registered} but not fully processed. */ /** Number of frames {@linkplain #registerInputFrame() registered} but not fully processed. */
private final AtomicInteger pendingFrameCount; private final AtomicInteger pendingFrameCount;
/** Wraps the {@link #inputSurfaceTexture}. */ /** Wraps the {@link #inputSurfaceTexture}. */
private final Surface inputSurface; private final Surface inputSurface;
/** Associated with an OpenGL external texture. */ /** Associated with an OpenGL external texture. */
private final SurfaceTexture inputSurfaceTexture; private final SurfaceTexture inputSurfaceTexture;
/** Identifier of the OpenGL texture associated with the input {@link SurfaceTexture}. */
private final int inputExternalTexId;
/** Transformation matrix associated with the {@link #inputSurfaceTexture}. */ /** Transformation matrix associated with the {@link #inputSurfaceTexture}. */
private final float[] textureTransformMatrix; private final float[] textureTransformMatrix;
...@@ -278,12 +281,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -278,12 +281,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* SingleFrameGlTextureProcessor SingleFrameGlTextureProcessors} at indices >= 1. * SingleFrameGlTextureProcessor SingleFrameGlTextureProcessors} at indices >= 1.
*/ */
private final ImmutableList<SingleFrameGlTextureProcessor> textureProcessors; private final ImmutableList<SingleFrameGlTextureProcessor> textureProcessors;
/** /**
* Identifiers of a framebuffer object associated with the intermediate textures that receive * {@link TextureInfo} instances describing the intermediate textures that receive output from the
* output from the previous {@link SingleFrameGlTextureProcessor}, and provide input for the * previous {@link SingleFrameGlTextureProcessor}, and provide input for the following {@link
* following {@link SingleFrameGlTextureProcessor}. * SingleFrameGlTextureProcessor}.
*/ */
private final int[] framebuffers; private final ImmutableList<TextureInfo> intermediateTextures;
/** The last texture processor's output {@link Size}. */
private final Size recommendedOutputSize;
private final Listener listener; private final Listener listener;
...@@ -318,8 +324,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -318,8 +324,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
ExecutorService singleThreadExecutorService, ExecutorService singleThreadExecutorService,
int inputExternalTexId, int inputExternalTexId,
long streamOffsetUs, long streamOffsetUs,
int[] framebuffers, ImmutableList<TextureInfo> intermediateTextures,
ImmutableList<SingleFrameGlTextureProcessor> textureProcessors, ImmutableList<SingleFrameGlTextureProcessor> textureProcessors,
Size recommendedOutputSize,
Listener listener, Listener listener,
boolean enableExperimentalHdrEditing) { boolean enableExperimentalHdrEditing) {
checkState(!textureProcessors.isEmpty()); checkState(!textureProcessors.isEmpty());
...@@ -327,9 +334,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -327,9 +334,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.eglDisplay = eglDisplay; this.eglDisplay = eglDisplay;
this.eglContext = eglContext; this.eglContext = eglContext;
this.singleThreadExecutorService = singleThreadExecutorService; this.singleThreadExecutorService = singleThreadExecutorService;
this.inputExternalTexId = inputExternalTexId;
this.streamOffsetUs = streamOffsetUs; this.streamOffsetUs = streamOffsetUs;
this.framebuffers = framebuffers; this.intermediateTextures = intermediateTextures;
this.textureProcessors = textureProcessors; this.textureProcessors = textureProcessors;
this.recommendedOutputSize = recommendedOutputSize;
this.listener = listener; this.listener = listener;
this.stopProcessing = new AtomicBoolean(); this.stopProcessing = new AtomicBoolean();
this.enableExperimentalHdrEditing = enableExperimentalHdrEditing; this.enableExperimentalHdrEditing = enableExperimentalHdrEditing;
...@@ -350,7 +359,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -350,7 +359,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* SurfaceView) output surface}. * SurfaceView) output surface}.
*/ */
public Size getOutputSize() { public Size getOutputSize() {
return getLast(textureProcessors).getOutputSize(); return recommendedOutputSize;
} }
/** /**
...@@ -493,37 +502,40 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -493,37 +502,40 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
inputSurfaceTexture.getTransformMatrix(textureTransformMatrix); inputSurfaceTexture.getTransformMatrix(textureTransformMatrix);
((ExternalTextureProcessor) textureProcessors.get(0)) ((ExternalTextureProcessor) textureProcessors.get(0))
.setTextureTransformMatrix(textureTransformMatrix); .setTextureTransformMatrix(textureTransformMatrix);
int inputTexId = inputExternalTexId;
for (int i = 0; i < textureProcessors.size() - 1; i++) { for (int i = 0; i < textureProcessors.size() - 1; i++) {
if (stopProcessing.get()) { if (stopProcessing.get()) {
return; return;
} }
Size intermediateSize = textureProcessors.get(i).getOutputSize(); TextureInfo outputTexture = intermediateTextures.get(i);
GlUtil.focusFramebuffer( GlUtil.focusFramebuffer(
eglDisplay, eglDisplay,
eglContext, eglContext,
outputEglSurface, outputEglSurface,
framebuffers[i], outputTexture.fboId,
intermediateSize.getWidth(), outputTexture.width,
intermediateSize.getHeight()); outputTexture.height);
clearOutputFrame(); clearOutputFrame();
textureProcessors.get(i).drawFrame(presentationTimeUs); textureProcessors.get(i).drawFrame(inputTexId, presentationTimeUs);
inputTexId = outputTexture.texId;
} }
GlUtil.focusEglSurface(eglDisplay, eglContext, outputEglSurface, outputWidth, outputHeight); GlUtil.focusEglSurface(eglDisplay, eglContext, outputEglSurface, outputWidth, outputHeight);
clearOutputFrame(); clearOutputFrame();
getLast(textureProcessors).drawFrame(presentationTimeUs); getLast(textureProcessors).drawFrame(inputTexId, presentationTimeUs);
EGLExt.eglPresentationTimeANDROID(eglDisplay, outputEglSurface, inputFrameTimeNs); EGLExt.eglPresentationTimeANDROID(eglDisplay, outputEglSurface, inputFrameTimeNs);
EGL14.eglSwapBuffers(eglDisplay, outputEglSurface); EGL14.eglSwapBuffers(eglDisplay, outputEglSurface);
if (debugSurfaceViewWrapper != null) { if (debugSurfaceViewWrapper != null) {
long framePresentationTimeUs = presentationTimeUs; long finalPresentationTimeUs = presentationTimeUs;
int finalInputTexId = inputTexId;
debugSurfaceViewWrapper.maybeRenderToSurfaceView( debugSurfaceViewWrapper.maybeRenderToSurfaceView(
() -> { () -> {
clearOutputFrame(); clearOutputFrame();
try { try {
getLast(textureProcessors).drawFrame(framePresentationTimeUs); getLast(textureProcessors).drawFrame(finalInputTexId, finalPresentationTimeUs);
} catch (FrameProcessingException e) { } catch (FrameProcessingException e) {
Log.d(TAG, "Error rendering to debug preview", e); Log.d(TAG, "Error rendering to debug preview", e);
} }
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlEffect.java
...@@ -15,16 +15,19 @@ ...@@ -15,16 +15,19 @@
*/ */
package com.google.android.exoplayer2.transformer; package com.google.android.exoplayer2.transformer;
import android.content.Context;
import java.io.IOException;
/** /**
* Interface for a video frame effect with a {@link SingleFrameGlTextureProcessor} implementation. * Interface for a video frame effect with a {@link SingleFrameGlTextureProcessor} implementation.
* *
* <p>Implementations contain information specifying the effect and can be {@linkplain * <p>Implementations contain information specifying the effect and can be {@linkplain
* #toGlTextureProcessor() converted} to a {@link SingleFrameGlTextureProcessor} which applies the * #toGlTextureProcessor(Context) converted} to a {@link SingleFrameGlTextureProcessor} which
* effect. * applies the effect.
*/ */
public interface GlEffect { public interface GlEffect {
/** Returns a {@link SingleFrameGlTextureProcessor} that applies the effect. */ /** Returns a {@link SingleFrameGlTextureProcessor} that applies the effect. */
// TODO(b/227625423): use GlTextureProcessor here once this interface exists. // TODO(b/227625423): use GlTextureProcessor here once this interface exists.
SingleFrameGlTextureProcessor toGlTextureProcessor(); SingleFrameGlTextureProcessor toGlTextureProcessor(Context context) throws IOException;
} }
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlMatrixTransformation.java
...@@ -15,8 +15,10 @@ ...@@ -15,8 +15,10 @@
*/ */
package com.google.android.exoplayer2.transformer; package com.google.android.exoplayer2.transformer;
import android.content.Context;
import android.opengl.Matrix; import android.opengl.Matrix;
import android.util.Size; import android.util.Size;
import java.io.IOException;
/** /**
* Specifies a 4x4 transformation {@link Matrix} to apply in the vertex shader for each frame. * Specifies a 4x4 transformation {@link Matrix} to apply in the vertex shader for each frame.
...@@ -47,7 +49,7 @@ public interface GlMatrixTransformation extends GlEffect { ...@@ -47,7 +49,7 @@ public interface GlMatrixTransformation extends GlEffect {
float[] getGlMatrixArray(long presentationTimeUs); float[] getGlMatrixArray(long presentationTimeUs);
@Override @Override
default SingleFrameGlTextureProcessor toGlTextureProcessor() { default SingleFrameGlTextureProcessor toGlTextureProcessor(Context context) throws IOException {
return new MatrixTransformationProcessor(this); return new MatrixTransformationProcessor(context, this);
} }
} }
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MatrixTransformationProcessor.java
...@@ -17,7 +17,6 @@ package com.google.android.exoplayer2.transformer; ...@@ -17,7 +17,6 @@ package com.google.android.exoplayer2.transformer;
import static com.google.android.exoplayer2.util.Assertions.checkArgument; import static com.google.android.exoplayer2.util.Assertions.checkArgument;
import static com.google.android.exoplayer2.util.Assertions.checkState; import static com.google.android.exoplayer2.util.Assertions.checkState;
import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
import android.content.Context; import android.content.Context;
import android.opengl.GLES20; import android.opengl.GLES20;
...@@ -28,7 +27,6 @@ import com.google.android.exoplayer2.util.GlUtil; ...@@ -28,7 +27,6 @@ import com.google.android.exoplayer2.util.GlUtil;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import java.io.IOException; import java.io.IOException;
import java.util.Arrays; import java.util.Arrays;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** /**
* Applies a sequence of transformation matrices in the vertex shader, and copies input pixels into * Applies a sequence of transformation matrices in the vertex shader, and copies input pixels into
...@@ -82,37 +80,45 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -82,37 +80,45 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
*/ */
private ImmutableList<float[]> visiblePolygon; private ImmutableList<float[]> visiblePolygon;
private @MonotonicNonNull Size outputSize; private final GlProgram glProgram;
private @MonotonicNonNull GlProgram glProgram;
/** /**
* Creates a new instance. * Creates a new instance.
* *
* @param context The {@link Context}.
* @param matrixTransformation A {@link MatrixTransformation} that specifies the transformation * @param matrixTransformation A {@link MatrixTransformation} that specifies the transformation
* matrix to use for each frame. * matrix to use for each frame.
* @throws IOException If a problem occurs while reading shader files.
*/ */
public MatrixTransformationProcessor(MatrixTransformation matrixTransformation) { public MatrixTransformationProcessor(Context context, MatrixTransformation matrixTransformation)
this(ImmutableList.of(matrixTransformation)); throws IOException {
this(context, ImmutableList.of(matrixTransformation));
} }
/** /**
* Creates a new instance. * Creates a new instance.
* *
* @param context The {@link Context}.
* @param matrixTransformation A {@link GlMatrixTransformation} that specifies the transformation * @param matrixTransformation A {@link GlMatrixTransformation} that specifies the transformation
* matrix to use for each frame. * matrix to use for each frame.
* @throws IOException If a problem occurs while reading shader files.
*/ */
public MatrixTransformationProcessor(GlMatrixTransformation matrixTransformation) { public MatrixTransformationProcessor(Context context, GlMatrixTransformation matrixTransformation)
this(ImmutableList.of(matrixTransformation)); throws IOException {
this(context, ImmutableList.of(matrixTransformation));
} }
/** /**
* Creates a new instance. * Creates a new instance.
* *
* @param context The {@link Context}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to * @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order. * apply to each frame in order.
* @throws IOException If a problem occurs while reading shader files.
*/ */
public MatrixTransformationProcessor( public MatrixTransformationProcessor(
ImmutableList<GlMatrixTransformation> matrixTransformations) { Context context, ImmutableList<GlMatrixTransformation> matrixTransformations)
throws IOException {
this.matrixTransformations = matrixTransformations; this.matrixTransformations = matrixTransformations;
transformationMatrixCache = new float[matrixTransformations.size()][16]; transformationMatrixCache = new float[matrixTransformations.size()][16];
...@@ -120,38 +126,33 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -120,38 +126,33 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
tempResultMatrix = new float[16]; tempResultMatrix = new float[16];
Matrix.setIdentityM(compositeTransformationMatrix, /* smOffset= */ 0); Matrix.setIdentityM(compositeTransformationMatrix, /* smOffset= */ 0);
visiblePolygon = NDC_SQUARE; visiblePolygon = NDC_SQUARE;
glProgram = new GlProgram(context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_PATH);
} }
@Override @Override
public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight) public Size configure(int inputWidth, int inputHeight) {
throws IOException {
checkArgument(inputWidth > 0, "inputWidth must be positive"); checkArgument(inputWidth > 0, "inputWidth must be positive");
checkArgument(inputHeight > 0, "inputHeight must be positive"); checkArgument(inputHeight > 0, "inputHeight must be positive");
outputSize = new Size(inputWidth, inputHeight); Size outputSize = new Size(inputWidth, inputHeight);
for (int i = 0; i < matrixTransformations.size(); i++) { for (int i = 0; i < matrixTransformations.size(); i++) {
outputSize = outputSize =
matrixTransformations.get(i).configure(outputSize.getWidth(), outputSize.getHeight()); matrixTransformations.get(i).configure(outputSize.getWidth(), outputSize.getHeight());
} }
glProgram = new GlProgram(context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_PATH); return outputSize;
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
}
@Override
public Size getOutputSize() {
return checkStateNotNull(outputSize);
} }
@Override @Override
public void drawFrame(long presentationTimeUs) throws FrameProcessingException { public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
updateCompositeTransformationMatrixAndVisiblePolygon(presentationTimeUs); updateCompositeTransformationMatrixAndVisiblePolygon(presentationTimeUs);
if (visiblePolygon.size() < 3) { if (visiblePolygon.size() < 3) {
return; // Need at least three visible vertices for a triangle. return; // Need at least three visible vertices for a triangle.
} }
try { try {
checkStateNotNull(glProgram).use(); glProgram.use();
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
glProgram.setFloatsUniform("uTransformationMatrix", compositeTransformationMatrix); glProgram.setFloatsUniform("uTransformationMatrix", compositeTransformationMatrix);
glProgram.setBufferAttribute( glProgram.setBufferAttribute(
"aFramePosition", "aFramePosition",
...@@ -168,10 +169,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -168,10 +169,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@Override @Override
public void release() { public void release() {
if (glProgram != null) {
glProgram.delete(); glProgram.delete();
} }
}
/** /**
* Updates {@link #compositeTransformationMatrix} and {@link #visiblePolygon} based on the given * Updates {@link #compositeTransformationMatrix} and {@link #visiblePolygon} based on the given
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SingleFrameGlTextureProcessor.java
...@@ -15,9 +15,7 @@ ...@@ -15,9 +15,7 @@
*/ */
package com.google.android.exoplayer2.transformer; package com.google.android.exoplayer2.transformer;
import android.content.Context;
import android.util.Size; import android.util.Size;
import java.io.IOException;
/** /**
* Manages a GLSL shader program for processing a frame. Implementations generally copy input pixels * Manages a GLSL shader program for processing a frame. Implementations generally copy input pixels
...@@ -26,53 +24,44 @@ import java.io.IOException; ...@@ -26,53 +24,44 @@ import java.io.IOException;
* <p>Methods must be called in the following order: * <p>Methods must be called in the following order:
* *
* <ol> * <ol>
* <li>The constructor, for implementation-specific arguments. * <li>{@link #configure(int, int)}, to configure the frame processor based on the input
* <li>{@link #initialize(Context, int, int, int)}, to set up graphics initialization. * dimensions.
* <li>{@link #drawFrame(long)}, to process one frame. * <li>{@link #drawFrame(int, long)}, to process one frame.
* <li>{@link #release()}, upon conclusion of processing. * <li>{@link #release()}, upon conclusion of processing.
* </ol> * </ol>
*
* <p>All methods in this class must be called on the thread that owns the OpenGL context.
*/ */
// TODO(b/227625423): Add GlTextureProcessor interface for async texture processors and make this an // TODO(b/227625423): Add GlTextureProcessor interface for async texture processors and make this an
// abstract class with a default implementation of GlTextureProcessor methods. // abstract class with a default implementation of GlTextureProcessor methods.
public interface SingleFrameGlTextureProcessor { public interface SingleFrameGlTextureProcessor {
/** /**
* Performs all initialization that requires OpenGL, such as, loading and compiling a GLSL shader * Configures the texture processor based on the input dimensions.
* program.
* *
* <p>This method may only be called if there is a current OpenGL context. * <p>This method can be called multiple times.
* *
* @param context The {@link Context}.
* @param inputTexId Identifier of a 2D OpenGL texture.
* @param inputWidth The input width, in pixels. * @param inputWidth The input width, in pixels.
* @param inputHeight The input height, in pixels. * @param inputHeight The input height, in pixels.
* @throws IOException If an error occurs while reading resources. * @return The output {@link Size} of frames processed through {@link #drawFrame(int, long)}.
*/
void initialize(Context context, int inputTexId, int inputWidth, int inputHeight)
throws IOException;
/**
* Returns the output {@link Size} of frames processed through {@link #drawFrame(long)}.
*
* <p>This method may only be called after the texture processor has been {@link
* #initialize(Context, int, int, int) initialized}.
*/ */
Size getOutputSize(); Size configure(int inputWidth, int inputHeight);
/** /**
* Draws one frame. * Draws one frame.
* *
* <p>This method may only be called after the texture processor has been {@link * <p>This method may only be called after the texture processor has been {@link #configure(int,
* #initialize(Context, int, int, int) initialized}. The caller is responsible for focussing the * int) configured}. The caller is responsible for focussing the correct render target before
* correct render target before calling this method. * calling this method.
* *
* <p>A minimal implementation should tell OpenGL to use its shader program, bind the shader * <p>A minimal implementation should tell OpenGL to use its shader program, bind the shader
* program's vertex attributes and uniforms, and issue a drawing command. * program's vertex attributes and uniforms, and issue a drawing command.
* *
* @param inputTexId Identifier of a 2D OpenGL texture containing the input frame.
* @param presentationTimeUs The presentation timestamp of the current frame, in microseconds. * @param presentationTimeUs The presentation timestamp of the current frame, in microseconds.
* @throws FrameProcessingException If an error occurs while processing or drawing the frame. * @throws FrameProcessingException If an error occurs while processing or drawing the frame.
*/ */
void drawFrame(long presentationTimeUs) throws FrameProcessingException; void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException;
/** Releases all resources. */ /** Releases all resources. */
void release(); void release();
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TextureInfo.java 0 → 100644
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.transformer;
/** Contains information describing an OpenGL texture. */
/* package */ final class TextureInfo {
/** The OpenGL texture identifier. */
public final int texId;
/** Identifier of a framebuffer object associated with the texture. */
public final int fboId;
/** The width of the texture, in pixels. */
public final int width;
/** The height of the texture, in pixels. */
public final int height;
/**
* Creates a new instance.
*
* @param texId The OpenGL texture identifier.
* @param fboId Identifier of a framebuffer object associated with the texture.
* @param width The width of the texture, in pixels.
* @param height The height of the texture, in pixels.
*/
public TextureInfo(int texId, int fboId, int width, int height) {
this.texId = texId;
this.fboId = fboId;
this.width = width;
this.height = height;
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment