Skip to content

SDK / exoplayer

Go to a project
Commit cd6ba068 by andrewlewis Committed by kim-vde
Options

Encapsulate attributes and uniforms within Program 

Document that apps should retain `GlUtil.Program` while the program is in use,
and keep a reference to attributes/uniforms within the program to make sure
they don't get GC'd causing any allocated buffers passed to GL to become
invalid.

Tested manually by running gldemo and transformer.

PiperOrigin-RevId: 411516894
parent b0cfe910
Expand all
Inline Side-by-side
Showing with 82 additions and 157 deletions
demos/gl/src/main/java/com/google/android/exoplayer2/gldemo/BitmapOverlayVideoProcessor.java
......@@ -26,7 +26,6 @@ import android.graphics.Paint;
import android.graphics.drawable.BitmapDrawable;
import android.opengl.GLES20;
import android.opengl.GLUtils;
import androidx.annotation.Nullable;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.util.GlUtil;
import java.io.IOException;
......@@ -52,8 +51,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final Canvas overlayCanvas;
private GlUtil.@MonotonicNonNull Program program;
@Nullable private GlUtil.Attribute[] attributes;
@Nullable private GlUtil.Uniform[] uniforms;
private float bitmapScaleX;
private float bitmapScaleY;
......@@ -88,31 +85,24 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} catch (IOException e) {
throw new IllegalStateException(e);
}
program.use();
GlUtil.Attribute[] attributes = program.getAttributes();
for (GlUtil.Attribute attribute : attributes) {
if (attribute.name.equals("a_position")) {
attribute.setBuffer(
new float[] {
-1, -1, 0, 1,
1, -1, 0, 1,
-1, 1, 0, 1,
1, 1, 0, 1
},
4);
} else if (attribute.name.equals("a_texcoord")) {
attribute.setBuffer(
new float[] {
0, 0, 0, 1,
1, 0, 0, 1,
0, 1, 0, 1,
1, 1, 0, 1
},
4);
}
}
this.attributes = attributes;
this.uniforms = program.getUniforms();
program.setBufferAttribute(
"a_position",
new float[] {
-1, -1, 0, 1,
1, -1, 0, 1,
-1, 1, 0, 1,
1, 1, 0, 1
},
4);
program.setBufferAttribute(
"a_texcoord",
new float[] {
0, 0, 0, 1,
1, 0, 0, 1,
0, 1, 0, 1,
1, 1, 0, 1
},
4);
GLES20.glGenTextures(1, textures, 0);
GLES20.glBindTexture(GL10.GL_TEXTURE_2D, textures[0]);
GLES20.glTexParameterf(GL10.GL_TEXTURE_2D, GL10.GL_TEXTURE_MIN_FILTER, GL10.GL_NEAREST);
......@@ -141,36 +131,22 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
GlUtil.checkGlError();
// Run the shader program.
GlUtil.Uniform[] uniforms = checkNotNull(this.uniforms);
GlUtil.Attribute[] attributes = checkNotNull(this.attributes);
for (GlUtil.Uniform uniform : uniforms) {
switch (uniform.name) {
case "tex_sampler_0":
uniform.setSamplerTexId(frameTexture, /* unit= */ 0);
break;
case "tex_sampler_1":
uniform.setSamplerTexId(textures[0], /* unit= */ 1);
break;
case "scaleX":
uniform.setFloat(bitmapScaleX);
break;
case "scaleY":
uniform.setFloat(bitmapScaleY);
break;
case "tex_transform":
uniform.setFloats(transformMatrix);
break;
default: // fall out
}
}
for (GlUtil.Attribute copyExternalAttribute : attributes) {
copyExternalAttribute.bind();
}
for (GlUtil.Uniform copyExternalUniform : uniforms) {
copyExternalUniform.bind();
}
GlUtil.Program program = checkNotNull(this.program);
program.setSamplerTexIdUniform("tex_sampler_0", frameTexture, /* unit= */ 0);
program.setSamplerTexIdUniform("tex_sampler_1", textures[0], /* unit= */ 1);
program.setFloatUniform("scaleX", bitmapScaleX);
program.setFloatUniform("scaleY", bitmapScaleY);
program.setFloatsUniform("tex_transform", transformMatrix);
program.bindAttributesAndUniforms();
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
GlUtil.checkGlError();
}
@Override
public void release() {
if (program != null) {
program.delete();
}
}
}
demos/gl/src/main/java/com/google/android/exoplayer2/gldemo/VideoProcessingGLSurfaceView.java
......@@ -64,6 +64,9 @@ public final class VideoProcessingGLSurfaceView extends GLSurfaceView {
* @param transformMatrix The 4 * 4 transform matrix to be applied to the texture.
*/
void draw(int frameTexture, long frameTimestampUs, float[] transformMatrix);
/** Releases any resources associated with this {@link VideoProcessor}. */
void release();
}
private static final int EGL_PROTECTED_CONTENT_EXT = 0x32C0;
......
library/core/src/main/java/com/google/android/exoplayer2/video/spherical/ProjectionRenderer.java
......@@ -114,7 +114,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
/** Initializes of the GL components. */
/* package */ void init() {
public void init() {
program = new GlUtil.Program(VERTEX_SHADER, FRAGMENT_SHADER);
mvpMatrixHandle = program.getUniformLocation("uMvpMatrix");
uTexMatrixHandle = program.getUniformLocation("uTexMatrix");
......@@ -132,7 +132,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* @param rightEye Whether the right eye view should be drawn. If {@code false}, the left eye view
* is drawn.
*/
/* package */ void draw(int textureId, float[] mvpMatrix, boolean rightEye) {
public void draw(int textureId, float[] mvpMatrix, boolean rightEye) {
MeshData meshData = rightEye ? rightMeshData : leftMeshData;
if (meshData == null) {
return;
......@@ -188,7 +188,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
/** Cleans up GL resources. */
/* package */ void shutdown() {
public void shutdown() {
if (program != null) {
program.delete();
}
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/OpenGlFrameEditor.java
......@@ -13,12 +13,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.transformer;
import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import static com.google.android.exoplayer2.util.Assertions.checkState;
import android.content.Context;
import android.graphics.SurfaceTexture;
import android.opengl.EGL14;
......@@ -31,7 +27,6 @@ import android.view.Surface;
import androidx.annotation.RequiresApi;
import com.google.android.exoplayer2.util.GlUtil;
import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* OpenGlFrameEditor applies changes to individual video frames using OpenGL. Changes include just
......@@ -67,73 +62,38 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
int textureId = GlUtil.createExternalTexture();
GlUtil.Program copyProgram;
try {
// TODO(internal b/205002913): check the loaded program is consistent with the attributes
// and uniforms expected in the code.
copyProgram = new GlUtil.Program(context, VERTEX_SHADER_FILE_PATH, FRAGMENT_SHADER_FILE_PATH);
} catch (IOException e) {
throw new IllegalStateException(e);
}
copyProgram.use();
GlUtil.Attribute[] copyAttributes = copyProgram.getAttributes();
checkState(
copyAttributes.length == EXPECTED_NUMBER_OF_ATTRIBUTES,
"Expected program to have " + EXPECTED_NUMBER_OF_ATTRIBUTES + " vertex attributes.");
for (GlUtil.Attribute copyAttribute : copyAttributes) {
if (copyAttribute.name.equals("a_position")) {
copyAttribute.setBuffer(
new float[] {
-1.0f, -1.0f, 0.0f, 1.0f,
1.0f, -1.0f, 0.0f, 1.0f,
-1.0f, 1.0f, 0.0f, 1.0f,
1.0f, 1.0f, 0.0f, 1.0f,
},
/* size= */ 4);
} else if (copyAttribute.name.equals("a_texcoord")) {
copyAttribute.setBuffer(
new float[] {
0.0f, 0.0f, 0.0f, 1.0f,
1.0f, 0.0f, 0.0f, 1.0f,
0.0f, 1.0f, 0.0f, 1.0f,
1.0f, 1.0f, 0.0f, 1.0f,
},
/* size= */ 4);
} else {
throw new IllegalStateException("Unexpected attribute name.");
}
copyAttribute.bind();
}
GlUtil.Uniform[] copyUniforms = copyProgram.getUniforms();
checkState(
copyUniforms.length == EXPECTED_NUMBER_OF_UNIFORMS,
"Expected program to have " + EXPECTED_NUMBER_OF_UNIFORMS + " uniforms.");
GlUtil.@MonotonicNonNull Uniform textureTransformUniform = null;
for (GlUtil.Uniform copyUniform : copyUniforms) {
if (copyUniform.name.equals("tex_sampler")) {
copyUniform.setSamplerTexId(textureId, 0);
copyUniform.bind();
} else if (copyUniform.name.equals("tex_transform")) {
textureTransformUniform = copyUniform;
} else {
throw new IllegalStateException("Unexpected uniform name.");
}
}
return new OpenGlFrameEditor(
eglDisplay,
eglContext,
eglSurface,
textureId,
checkNotNull(textureTransformUniform),
copyProgram,
copyAttributes,
copyUniforms);
copyProgram.setBufferAttribute(
"a_position",
new float[] {
-1.0f, -1.0f, 0.0f, 1.0f,
1.0f, -1.0f, 0.0f, 1.0f,
-1.0f, 1.0f, 0.0f, 1.0f,
1.0f, 1.0f, 0.0f, 1.0f,
},
/* size= */ 4);
copyProgram.setBufferAttribute(
"a_texcoord",
new float[] {
0.0f, 0.0f, 0.0f, 1.0f,
1.0f, 0.0f, 0.0f, 1.0f,
0.0f, 1.0f, 0.0f, 1.0f,
1.0f, 1.0f, 0.0f, 1.0f,
},
/* size= */ 4);
copyProgram.setSamplerTexIdUniform("tex_sampler", textureId, /* unit= */ 0);
return new OpenGlFrameEditor(eglDisplay, eglContext, eglSurface, textureId, copyProgram);
}
// Predefined shader values.
private static final String VERTEX_SHADER_FILE_PATH = "shaders/blit_vertex_shader.glsl";
private static final String FRAGMENT_SHADER_FILE_PATH =
"shaders/copy_external_fragment_shader.glsl";
private static final int EXPECTED_NUMBER_OF_ATTRIBUTES = 2;
private static final int EXPECTED_NUMBER_OF_UNIFORMS = 2;
private final float[] textureTransformMatrix;
private final EGLDisplay eglDisplay;
......@@ -142,19 +102,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final int textureId;
private final SurfaceTexture inputSurfaceTexture;
private final Surface inputSurface;
private final GlUtil.Uniform textureTransformUniform;
// TODO(internal: b/206631334): These fields ensure buffers passed to GL are not GC'ed. Implement
// a better way of doing this so they aren't just unused fields.
@SuppressWarnings("unused")
private final GlUtil.Program copyProgram;
@SuppressWarnings("unused")
private final GlUtil.Attribute[] copyAttributes;
@SuppressWarnings("unused")
private final GlUtil.Uniform[] copyUniforms;
private volatile boolean hasInputData;
private OpenGlFrameEditor(
......@@ -162,38 +112,37 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
EGLContext eglContext,
EGLSurface eglSurface,
int textureId,
GlUtil.Uniform textureTransformUniform,
GlUtil.Program copyProgram,
GlUtil.Attribute[] copyAttributes,
GlUtil.Uniform[] copyUniforms) {
GlUtil.Program copyProgram) {
this.eglDisplay = eglDisplay;
this.eglContext = eglContext;
this.eglSurface = eglSurface;
this.textureId = textureId;
this.textureTransformUniform = textureTransformUniform;
this.copyProgram = copyProgram;
this.copyAttributes = copyAttributes;
this.copyUniforms = copyUniforms;
textureTransformMatrix = new float[16];
inputSurfaceTexture = new SurfaceTexture(textureId);
inputSurfaceTexture.setOnFrameAvailableListener(surfaceTexture -> hasInputData = true);
inputSurface = new Surface(inputSurfaceTexture);
}
/** Releases all resources. */
public void release() {
GlUtil.destroyEglContext(eglDisplay, eglContext);
GlUtil.deleteTexture(textureId);
inputSurfaceTexture.release();
inputSurface.release();
/** Returns the input {@link Surface}. */
public Surface getInputSurface() {
return inputSurface;
}
/**
* Returns whether there is pending input data that can be processed by calling {@link
* #processData()}.
*/
public boolean hasInputData() {
return hasInputData;
}
/** Informs the editor that there is new input data available for it to process asynchronously. */
/** Processes pending input data. */
public void processData() {
inputSurfaceTexture.updateTexImage();
inputSurfaceTexture.getTransformMatrix(textureTransformMatrix);
textureTransformUniform.setFloats(textureTransformMatrix);
textureTransformUniform.bind();
copyProgram.setFloatsUniform("tex_transform", textureTransformMatrix);
copyProgram.bindAttributesAndUniforms();
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
long surfaceTextureTimestampNs = inputSurfaceTexture.getTimestamp();
EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, surfaceTextureTimestampNs);
......@@ -201,15 +150,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
hasInputData = false;
}
/**
* Returns the input {@link Surface} after configuring the editor if it has not previously been
* configured.
*/
public Surface getInputSurface() {
return inputSurface;
}
public boolean hasInputData() {
return hasInputData;
/** Releases all resources. */
public void release() {
copyProgram.delete();
GlUtil.deleteTexture(textureId);
GlUtil.destroyEglContext(eglDisplay, eglContext);
inputSurfaceTexture.release();
inputSurface.release();
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment