Skip to content

SDK / exoplayer

Go to a project
Commit cd6ba068 by andrewlewis Committed by kim-vde
Options

Encapsulate attributes and uniforms within Program 

Document that apps should retain `GlUtil.Program` while the program is in use,
and keep a reference to attributes/uniforms within the program to make sure
they don't get GC'd causing any allocated buffers passed to GL to become
invalid.

Tested manually by running gldemo and transformer.

PiperOrigin-RevId: 411516894
parent b0cfe910
Expand all
Inline Side-by-side
Showing with 54 additions and 129 deletions
demos/gl/src/main/java/com/google/android/exoplayer2/gldemo/BitmapOverlayVideoProcessor.java
...@@ -26,7 +26,6 @@ import android.graphics.Paint; ...@@ -26,7 +26,6 @@ import android.graphics.Paint;
import android.graphics.drawable.BitmapDrawable; import android.graphics.drawable.BitmapDrawable;
import android.opengl.GLES20; import android.opengl.GLES20;
import android.opengl.GLUtils; import android.opengl.GLUtils;
import androidx.annotation.Nullable;
import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.util.GlUtil; import com.google.android.exoplayer2.util.GlUtil;
import java.io.IOException; import java.io.IOException;
...@@ -52,8 +51,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -52,8 +51,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final Canvas overlayCanvas; private final Canvas overlayCanvas;
private GlUtil.@MonotonicNonNull Program program; private GlUtil.@MonotonicNonNull Program program;
@Nullable private GlUtil.Attribute[] attributes;
@Nullable private GlUtil.Uniform[] uniforms;
private float bitmapScaleX; private float bitmapScaleX;
private float bitmapScaleY; private float bitmapScaleY;
...@@ -88,11 +85,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -88,11 +85,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} catch (IOException e) { } catch (IOException e) {
throw new IllegalStateException(e); throw new IllegalStateException(e);
} }
program.use(); program.setBufferAttribute(
GlUtil.Attribute[] attributes = program.getAttributes(); "a_position",
for (GlUtil.Attribute attribute : attributes) {
if (attribute.name.equals("a_position")) {
attribute.setBuffer(
new float[] { new float[] {
-1, -1, 0, 1, -1, -1, 0, 1,
1, -1, 0, 1, 1, -1, 0, 1,
...@@ -100,8 +94,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -100,8 +94,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
1, 1, 0, 1 1, 1, 0, 1
}, },
4); 4);
} else if (attribute.name.equals("a_texcoord")) { program.setBufferAttribute(
attribute.setBuffer( "a_texcoord",
new float[] { new float[] {
0, 0, 0, 1, 0, 0, 0, 1,
1, 0, 0, 1, 1, 0, 0, 1,
...@@ -109,10 +103,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -109,10 +103,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
1, 1, 0, 1 1, 1, 0, 1
}, },
4); 4);
}
}
this.attributes = attributes;
this.uniforms = program.getUniforms();
GLES20.glGenTextures(1, textures, 0); GLES20.glGenTextures(1, textures, 0);
GLES20.glBindTexture(GL10.GL_TEXTURE_2D, textures[0]); GLES20.glBindTexture(GL10.GL_TEXTURE_2D, textures[0]);
GLES20.glTexParameterf(GL10.GL_TEXTURE_2D, GL10.GL_TEXTURE_MIN_FILTER, GL10.GL_NEAREST); GLES20.glTexParameterf(GL10.GL_TEXTURE_2D, GL10.GL_TEXTURE_MIN_FILTER, GL10.GL_NEAREST);
...@@ -141,36 +131,22 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -141,36 +131,22 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
GlUtil.checkGlError(); GlUtil.checkGlError();
// Run the shader program. // Run the shader program.
GlUtil.Uniform[] uniforms = checkNotNull(this.uniforms); GlUtil.Program program = checkNotNull(this.program);
GlUtil.Attribute[] attributes = checkNotNull(this.attributes); program.setSamplerTexIdUniform("tex_sampler_0", frameTexture, /* unit= */ 0);
for (GlUtil.Uniform uniform : uniforms) { program.setSamplerTexIdUniform("tex_sampler_1", textures[0], /* unit= */ 1);
switch (uniform.name) { program.setFloatUniform("scaleX", bitmapScaleX);
case "tex_sampler_0": program.setFloatUniform("scaleY", bitmapScaleY);
uniform.setSamplerTexId(frameTexture, /* unit= */ 0); program.setFloatsUniform("tex_transform", transformMatrix);
break; program.bindAttributesAndUniforms();
case "tex_sampler_1":
uniform.setSamplerTexId(textures[0], /* unit= */ 1);
break;
case "scaleX":
uniform.setFloat(bitmapScaleX);
break;
case "scaleY":
uniform.setFloat(bitmapScaleY);
break;
case "tex_transform":
uniform.setFloats(transformMatrix);
break;
default: // fall out
}
}
for (GlUtil.Attribute copyExternalAttribute : attributes) {
copyExternalAttribute.bind();
}
for (GlUtil.Uniform copyExternalUniform : uniforms) {
copyExternalUniform.bind();
}
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
GlUtil.checkGlError(); GlUtil.checkGlError();
} }
@Override
public void release() {
if (program != null) {
program.delete();
}
}
} }
demos/gl/src/main/java/com/google/android/exoplayer2/gldemo/VideoProcessingGLSurfaceView.java
...@@ -64,6 +64,9 @@ public final class VideoProcessingGLSurfaceView extends GLSurfaceView { ...@@ -64,6 +64,9 @@ public final class VideoProcessingGLSurfaceView extends GLSurfaceView {
* @param transformMatrix The 4 * 4 transform matrix to be applied to the texture. * @param transformMatrix The 4 * 4 transform matrix to be applied to the texture.
*/ */
void draw(int frameTexture, long frameTimestampUs, float[] transformMatrix); void draw(int frameTexture, long frameTimestampUs, float[] transformMatrix);
/** Releases any resources associated with this {@link VideoProcessor}. */
void release();
} }
private static final int EGL_PROTECTED_CONTENT_EXT = 0x32C0; private static final int EGL_PROTECTED_CONTENT_EXT = 0x32C0;
......
library/core/src/main/java/com/google/android/exoplayer2/video/spherical/ProjectionRenderer.java
...@@ -114,7 +114,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -114,7 +114,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
/** Initializes of the GL components. */ /** Initializes of the GL components. */
/* package */ void init() { public void init() {
program = new GlUtil.Program(VERTEX_SHADER, FRAGMENT_SHADER); program = new GlUtil.Program(VERTEX_SHADER, FRAGMENT_SHADER);
mvpMatrixHandle = program.getUniformLocation("uMvpMatrix"); mvpMatrixHandle = program.getUniformLocation("uMvpMatrix");
uTexMatrixHandle = program.getUniformLocation("uTexMatrix"); uTexMatrixHandle = program.getUniformLocation("uTexMatrix");
...@@ -132,7 +132,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -132,7 +132,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* @param rightEye Whether the right eye view should be drawn. If {@code false}, the left eye view * @param rightEye Whether the right eye view should be drawn. If {@code false}, the left eye view
* is drawn. * is drawn.
*/ */
/* package */ void draw(int textureId, float[] mvpMatrix, boolean rightEye) { public void draw(int textureId, float[] mvpMatrix, boolean rightEye) {
MeshData meshData = rightEye ? rightMeshData : leftMeshData; MeshData meshData = rightEye ? rightMeshData : leftMeshData;
if (meshData == null) { if (meshData == null) {
return; return;
...@@ -188,7 +188,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -188,7 +188,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
/** Cleans up GL resources. */ /** Cleans up GL resources. */
/* package */ void shutdown() { public void shutdown() {
if (program != null) { if (program != null) {
program.delete(); program.delete();
} }
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/OpenGlFrameEditor.java
...@@ -13,12 +13,8 @@ ...@@ -13,12 +13,8 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package com.google.android.exoplayer2.transformer; package com.google.android.exoplayer2.transformer;
import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import static com.google.android.exoplayer2.util.Assertions.checkState;
import android.content.Context; import android.content.Context;
import android.graphics.SurfaceTexture; import android.graphics.SurfaceTexture;
import android.opengl.EGL14; import android.opengl.EGL14;
...@@ -31,7 +27,6 @@ import android.view.Surface; ...@@ -31,7 +27,6 @@ import android.view.Surface;
import androidx.annotation.RequiresApi; import androidx.annotation.RequiresApi;
import com.google.android.exoplayer2.util.GlUtil; import com.google.android.exoplayer2.util.GlUtil;
import java.io.IOException; import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** /**
* OpenGlFrameEditor applies changes to individual video frames using OpenGL. Changes include just * OpenGlFrameEditor applies changes to individual video frames using OpenGL. Changes include just
...@@ -67,19 +62,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -67,19 +62,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
int textureId = GlUtil.createExternalTexture(); int textureId = GlUtil.createExternalTexture();
GlUtil.Program copyProgram; GlUtil.Program copyProgram;
try { try {
// TODO(internal b/205002913): check the loaded program is consistent with the attributes
// and uniforms expected in the code.
copyProgram = new GlUtil.Program(context, VERTEX_SHADER_FILE_PATH, FRAGMENT_SHADER_FILE_PATH); copyProgram = new GlUtil.Program(context, VERTEX_SHADER_FILE_PATH, FRAGMENT_SHADER_FILE_PATH);
} catch (IOException e) { } catch (IOException e) {
throw new IllegalStateException(e); throw new IllegalStateException(e);
} }
copyProgram.setBufferAttribute(
copyProgram.use(); "a_position",
GlUtil.Attribute[] copyAttributes = copyProgram.getAttributes();
checkState(
copyAttributes.length == EXPECTED_NUMBER_OF_ATTRIBUTES,
"Expected program to have " + EXPECTED_NUMBER_OF_ATTRIBUTES + " vertex attributes.");
for (GlUtil.Attribute copyAttribute : copyAttributes) {
if (copyAttribute.name.equals("a_position")) {
copyAttribute.setBuffer(
new float[] { new float[] {
-1.0f, -1.0f, 0.0f, 1.0f, -1.0f, -1.0f, 0.0f, 1.0f,
1.0f, -1.0f, 0.0f, 1.0f, 1.0f, -1.0f, 0.0f, 1.0f,
...@@ -87,8 +77,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -87,8 +77,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
1.0f, 1.0f, 0.0f, 1.0f, 1.0f, 1.0f, 0.0f, 1.0f,
}, },
/* size= */ 4); /* size= */ 4);
} else if (copyAttribute.name.equals("a_texcoord")) { copyProgram.setBufferAttribute(
copyAttribute.setBuffer( "a_texcoord",
new float[] { new float[] {
0.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 1.0f,
1.0f, 0.0f, 0.0f, 1.0f, 1.0f, 0.0f, 0.0f, 1.0f,
...@@ -96,44 +86,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -96,44 +86,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
1.0f, 1.0f, 0.0f, 1.0f, 1.0f, 1.0f, 0.0f, 1.0f,
}, },
/* size= */ 4); /* size= */ 4);
} else { copyProgram.setSamplerTexIdUniform("tex_sampler", textureId, /* unit= */ 0);
throw new IllegalStateException("Unexpected attribute name."); return new OpenGlFrameEditor(eglDisplay, eglContext, eglSurface, textureId, copyProgram);
}
copyAttribute.bind();
}
GlUtil.Uniform[] copyUniforms = copyProgram.getUniforms();
checkState(
copyUniforms.length == EXPECTED_NUMBER_OF_UNIFORMS,
"Expected program to have " + EXPECTED_NUMBER_OF_UNIFORMS + " uniforms.");
GlUtil.@MonotonicNonNull Uniform textureTransformUniform = null;
for (GlUtil.Uniform copyUniform : copyUniforms) {
if (copyUniform.name.equals("tex_sampler")) {
copyUniform.setSamplerTexId(textureId, 0);
copyUniform.bind();
} else if (copyUniform.name.equals("tex_transform")) {
textureTransformUniform = copyUniform;
} else {
throw new IllegalStateException("Unexpected uniform name.");
}
}
return new OpenGlFrameEditor(
eglDisplay,
eglContext,
eglSurface,
textureId,
checkNotNull(textureTransformUniform),
copyProgram,
copyAttributes,
copyUniforms);
} }
// Predefined shader values. // Predefined shader values.
private static final String VERTEX_SHADER_FILE_PATH = "shaders/blit_vertex_shader.glsl"; private static final String VERTEX_SHADER_FILE_PATH = "shaders/blit_vertex_shader.glsl";
private static final String FRAGMENT_SHADER_FILE_PATH = private static final String FRAGMENT_SHADER_FILE_PATH =
"shaders/copy_external_fragment_shader.glsl"; "shaders/copy_external_fragment_shader.glsl";
private static final int EXPECTED_NUMBER_OF_ATTRIBUTES = 2;
private static final int EXPECTED_NUMBER_OF_UNIFORMS = 2;
private final float[] textureTransformMatrix; private final float[] textureTransformMatrix;
private final EGLDisplay eglDisplay; private final EGLDisplay eglDisplay;
...@@ -142,19 +102,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -142,19 +102,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final int textureId; private final int textureId;
private final SurfaceTexture inputSurfaceTexture; private final SurfaceTexture inputSurfaceTexture;
private final Surface inputSurface; private final Surface inputSurface;
private final GlUtil.Uniform textureTransformUniform;
// TODO(internal: b/206631334): These fields ensure buffers passed to GL are not GC'ed. Implement
// a better way of doing this so they aren't just unused fields.
@SuppressWarnings("unused")
private final GlUtil.Program copyProgram; private final GlUtil.Program copyProgram;
@SuppressWarnings("unused")
private final GlUtil.Attribute[] copyAttributes;
@SuppressWarnings("unused")
private final GlUtil.Uniform[] copyUniforms;
private volatile boolean hasInputData; private volatile boolean hasInputData;
private OpenGlFrameEditor( private OpenGlFrameEditor(
...@@ -162,38 +112,37 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -162,38 +112,37 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
EGLContext eglContext, EGLContext eglContext,
EGLSurface eglSurface, EGLSurface eglSurface,
int textureId, int textureId,
GlUtil.Uniform textureTransformUniform, GlUtil.Program copyProgram) {
GlUtil.Program copyProgram,
GlUtil.Attribute[] copyAttributes,
GlUtil.Uniform[] copyUniforms) {
this.eglDisplay = eglDisplay; this.eglDisplay = eglDisplay;
this.eglContext = eglContext; this.eglContext = eglContext;
this.eglSurface = eglSurface; this.eglSurface = eglSurface;
this.textureId = textureId; this.textureId = textureId;
this.textureTransformUniform = textureTransformUniform;
this.copyProgram = copyProgram; this.copyProgram = copyProgram;
this.copyAttributes = copyAttributes;
this.copyUniforms = copyUniforms;
textureTransformMatrix = new float[16]; textureTransformMatrix = new float[16];
inputSurfaceTexture = new SurfaceTexture(textureId); inputSurfaceTexture = new SurfaceTexture(textureId);
inputSurfaceTexture.setOnFrameAvailableListener(surfaceTexture -> hasInputData = true); inputSurfaceTexture.setOnFrameAvailableListener(surfaceTexture -> hasInputData = true);
inputSurface = new Surface(inputSurfaceTexture); inputSurface = new Surface(inputSurfaceTexture);
} }
/** Releases all resources. */ /** Returns the input {@link Surface}. */
public void release() { public Surface getInputSurface() {
GlUtil.destroyEglContext(eglDisplay, eglContext); return inputSurface;
GlUtil.deleteTexture(textureId); }
inputSurfaceTexture.release();
inputSurface.release(); /**
* Returns whether there is pending input data that can be processed by calling {@link
* #processData()}.
*/
public boolean hasInputData() {
return hasInputData;
} }
/** Informs the editor that there is new input data available for it to process asynchronously. */ /** Processes pending input data. */
public void processData() { public void processData() {
inputSurfaceTexture.updateTexImage(); inputSurfaceTexture.updateTexImage();
inputSurfaceTexture.getTransformMatrix(textureTransformMatrix); inputSurfaceTexture.getTransformMatrix(textureTransformMatrix);
textureTransformUniform.setFloats(textureTransformMatrix); copyProgram.setFloatsUniform("tex_transform", textureTransformMatrix);
textureTransformUniform.bind(); copyProgram.bindAttributesAndUniforms();
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
long surfaceTextureTimestampNs = inputSurfaceTexture.getTimestamp(); long surfaceTextureTimestampNs = inputSurfaceTexture.getTimestamp();
EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, surfaceTextureTimestampNs); EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, surfaceTextureTimestampNs);
...@@ -201,15 +150,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -201,15 +150,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
hasInputData = false; hasInputData = false;
} }
/** /** Releases all resources. */
* Returns the input {@link Surface} after configuring the editor if it has not previously been public void release() {
* configured. copyProgram.delete();
*/ GlUtil.deleteTexture(textureId);
public Surface getInputSurface() { GlUtil.destroyEglContext(eglDisplay, eglContext);
return inputSurface; inputSurfaceTexture.release();
} inputSurface.release();
public boolean hasInputData() {
return hasInputData;
} }
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment