Skip to content

SDK / exoplayer

Go to a project
Commit 2d1875fd by claincly Committed by Tianyi Feng
Options

Add a base class for GL shader programs. 

By having a single base class for GL shader programs we simplify the customization
of new shader programs. The concrete cases include

- Allow frame dropping in shader program
- Creating a FrameCache that selectively (based on timestamp) replays and clears
  the cached content

PiperOrigin-RevId: 520322060
parent 411fb9ea
Expand all
Inline Side-by-side
Showing with 263 additions and 335 deletions
library/effect/src/main/java/com/google/android/exoplayer2/effect/FrameCacheGlShaderProgram.java
...@@ -15,52 +15,33 @@ ...@@ -15,52 +15,33 @@
*/ */
package com.google.android.exoplayer2.effect; package com.google.android.exoplayer2.effect;
import static com.google.android.exoplayer2.util.Assertions.checkState;
import android.content.Context; import android.content.Context;
import android.opengl.GLES20; import android.opengl.GLES20;
import com.google.android.exoplayer2.util.GlObjectsProvider;
import com.google.android.exoplayer2.util.GlProgram; import com.google.android.exoplayer2.util.GlProgram;
import com.google.android.exoplayer2.util.GlTextureInfo; import com.google.android.exoplayer2.util.GlTextureInfo;
import com.google.android.exoplayer2.util.GlUtil; import com.google.android.exoplayer2.util.GlUtil;
import com.google.android.exoplayer2.util.Size;
import com.google.android.exoplayer2.util.VideoFrameProcessingException; import com.google.android.exoplayer2.util.VideoFrameProcessingException;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.MoreExecutors;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayDeque;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.concurrent.Executor;
/** /**
* Manages a pool of {@linkplain GlTextureInfo textures}, and caches the input frame. * Manages a pool of {@linkplain GlTextureInfo textures}, and caches the input frame.
* *
* <p>Implements {@link FrameCache}. * <p>Implements {@link FrameCache}.
*/ */
/* package */ final class FrameCacheGlShaderProgram implements GlShaderProgram { /* package */ final class FrameCacheGlShaderProgram extends BaseGlShaderProgram {
private static final String VERTEX_SHADER_TRANSFORMATION_ES2_PATH = private static final String VERTEX_SHADER_TRANSFORMATION_ES2_PATH =
"shaders/vertex_shader_transformation_es2.glsl"; "shaders/vertex_shader_transformation_es2.glsl";
private static final String FRAGMENT_SHADER_TRANSFORMATION_ES2_PATH = private static final String FRAGMENT_SHADER_TRANSFORMATION_ES2_PATH =
"shaders/fragment_shader_transformation_es2.glsl"; "shaders/fragment_shader_transformation_es2.glsl";
private final ArrayDeque<GlTextureInfo> freeOutputTextures;
private final ArrayDeque<GlTextureInfo> inUseOutputTextures;
private final GlProgram copyProgram; private final GlProgram copyProgram;
private final int capacity;
private final boolean useHdr;
private GlObjectsProvider glObjectsProvider;
private InputListener inputListener;
private OutputListener outputListener;
private ErrorListener errorListener;
private Executor errorListenerExecutor;
private boolean frameProcessingStarted;
/** Creates a new instance. */ /** Creates a new instance. */
public FrameCacheGlShaderProgram(Context context, int capacity, boolean useHdr) public FrameCacheGlShaderProgram(Context context, int capacity, boolean useHdr)
throws VideoFrameProcessingException { throws VideoFrameProcessingException {
freeOutputTextures = new ArrayDeque<>(); super(useHdr, capacity);
inUseOutputTextures = new ArrayDeque<>();
try { try {
this.copyProgram = this.copyProgram =
new GlProgram( new GlProgram(
...@@ -70,8 +51,6 @@ import java.util.concurrent.Executor; ...@@ -70,8 +51,6 @@ import java.util.concurrent.Executor;
} catch (IOException | GlUtil.GlException e) { } catch (IOException | GlUtil.GlException e) {
throw VideoFrameProcessingException.from(e); throw VideoFrameProcessingException.from(e);
} }
this.capacity = capacity;
this.useHdr = useHdr;
float[] identityMatrix = GlUtil.create4x4IdentityMatrix(); float[] identityMatrix = GlUtil.create4x4IdentityMatrix();
copyProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix); copyProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix);
...@@ -81,73 +60,17 @@ import java.util.concurrent.Executor; ...@@ -81,73 +60,17 @@ import java.util.concurrent.Executor;
"aFramePosition", "aFramePosition",
GlUtil.getNormalizedCoordinateBounds(), GlUtil.getNormalizedCoordinateBounds(),
GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
glObjectsProvider = GlObjectsProvider.DEFAULT;
inputListener = new InputListener() {};
outputListener = new OutputListener() {};
errorListener = videoFrameProcessingException -> {};
errorListenerExecutor = MoreExecutors.directExecutor();
}
@Override
public void setGlObjectsProvider(GlObjectsProvider glObjectsProvider) {
checkState(
!frameProcessingStarted,
"The GlObjectsProvider cannot be set after frame processing has started.");
this.glObjectsProvider = glObjectsProvider;
}
@Override
public void setInputListener(InputListener inputListener) {
this.inputListener = inputListener;
int numberOfFreeFramesToNotify;
if (getIteratorToAllTextures().hasNext()) {
// The frame buffers have already been allocated.
numberOfFreeFramesToNotify = freeOutputTextures.size();
} else {
// Defer frame buffer allocation to when queueing input frames.
numberOfFreeFramesToNotify = capacity;
}
for (int i = 0; i < numberOfFreeFramesToNotify; i++) {
inputListener.onReadyToAcceptInputFrame();
}
} }
@Override @Override
public void setOutputListener(OutputListener outputListener) { public Size configure(int inputWidth, int inputHeight) {
this.outputListener = outputListener; return new Size(inputWidth, inputHeight);
} }
@Override @Override
public void setErrorListener(Executor errorListenerExecutor, ErrorListener errorListener) { public void drawFrame(int inputTexId, long presentationTimeUs)
this.errorListenerExecutor = errorListenerExecutor; throws VideoFrameProcessingException {
this.errorListener = errorListener;
}
@Override
public void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) {
frameProcessingStarted = true;
try { try {
configureAllOutputTextures(inputTexture.width, inputTexture.height);
// Focus on the next free buffer.
GlTextureInfo outputTexture = freeOutputTextures.remove();
inUseOutputTextures.add(outputTexture);
// Copy frame to fbo.
GlUtil.focusFramebufferUsingCurrentContext(
outputTexture.fboId, outputTexture.width, outputTexture.height);
GlUtil.clearOutputFrame();
drawFrame(inputTexture.texId);
inputListener.onInputFrameProcessed(inputTexture);
outputListener.onOutputFrameAvailable(outputTexture, presentationTimeUs);
} catch (GlUtil.GlException | NoSuchElementException e) {
errorListenerExecutor.execute(
() -> errorListener.onError(VideoFrameProcessingException.from(e)));
}
}
private void drawFrame(int inputTexId) throws GlUtil.GlException {
copyProgram.use(); copyProgram.use();
copyProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); copyProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
copyProgram.bindAttributesAndUniforms(); copyProgram.bindAttributesAndUniforms();
...@@ -155,81 +78,8 @@ import java.util.concurrent.Executor; ...@@ -155,81 +78,8 @@ import java.util.concurrent.Executor;
GLES20.GL_TRIANGLE_STRIP, GLES20.GL_TRIANGLE_STRIP,
/* first= */ 0, /* first= */ 0,
/* count= */ GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); /* count= */ GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
}
@Override
public void releaseOutputFrame(GlTextureInfo outputTexture) {
frameProcessingStarted = true;
checkState(inUseOutputTextures.contains(outputTexture));
inUseOutputTextures.remove(outputTexture);
freeOutputTextures.add(outputTexture);
inputListener.onReadyToAcceptInputFrame();
}
@Override
public void signalEndOfCurrentInputStream() {
frameProcessingStarted = true;
outputListener.onCurrentOutputStreamEnded();
}
@Override
public void flush() {
frameProcessingStarted = true;
freeOutputTextures.addAll(inUseOutputTextures);
inUseOutputTextures.clear();
inputListener.onFlush();
for (int i = 0; i < freeOutputTextures.size(); i++) {
inputListener.onReadyToAcceptInputFrame();
}
}
@Override
public void release() throws VideoFrameProcessingException {
frameProcessingStarted = true;
try {
deleteAllOutputTextures();
} catch (GlUtil.GlException e) { } catch (GlUtil.GlException e) {
throw new VideoFrameProcessingException(e); throw VideoFrameProcessingException.from(e);
}
}
private void configureAllOutputTextures(int inputWidth, int inputHeight)
throws GlUtil.GlException {
Iterator<GlTextureInfo> allTextures = getIteratorToAllTextures();
if (!allTextures.hasNext()) {
createAllOutputTextures(inputWidth, inputHeight);
return;
}
GlTextureInfo outputGlTextureInfo = allTextures.next();
if (outputGlTextureInfo.width != inputWidth || outputGlTextureInfo.height != inputHeight) {
deleteAllOutputTextures();
createAllOutputTextures(inputWidth, inputHeight);
}
}
private void createAllOutputTextures(int width, int height) throws GlUtil.GlException {
checkState(freeOutputTextures.isEmpty());
checkState(inUseOutputTextures.isEmpty());
for (int i = 0; i < capacity; i++) {
int outputTexId = GlUtil.createTexture(width, height, useHdr);
GlTextureInfo outputTexture =
glObjectsProvider.createBuffersForTexture(outputTexId, width, height);
freeOutputTextures.add(outputTexture);
}
}
private void deleteAllOutputTextures() throws GlUtil.GlException {
Iterator<GlTextureInfo> allTextures = getIteratorToAllTextures();
while (allTextures.hasNext()) {
GlTextureInfo textureInfo = allTextures.next();
GlUtil.deleteTexture(textureInfo.texId);
GlUtil.deleteFbo(textureInfo.fboId);
}
freeOutputTextures.clear();
inUseOutputTextures.clear();
} }
private Iterator<GlTextureInfo> getIteratorToAllTextures() {
return Iterables.concat(freeOutputTextures, inUseOutputTextures).iterator();
} }
} }
library/effect/src/main/java/com/google/android/exoplayer2/effect/SingleFrameGlShaderProgram.java
...@@ -15,19 +15,6 @@ ...@@ -15,19 +15,6 @@
*/ */
package com.google.android.exoplayer2.effect; package com.google.android.exoplayer2.effect;
import static com.google.android.exoplayer2.util.Assertions.checkState;
import androidx.annotation.CallSuper;
import com.google.android.exoplayer2.util.GlObjectsProvider;
import com.google.android.exoplayer2.util.GlTextureInfo;
import com.google.android.exoplayer2.util.GlUtil;
import com.google.android.exoplayer2.util.Size;
import com.google.android.exoplayer2.util.VideoFrameProcessingException;
import com.google.common.util.concurrent.MoreExecutors;
import java.util.concurrent.Executor;
import org.checkerframework.checker.nullness.qual.EnsuresNonNull;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** /**
* Manages a GLSL shader program for processing a frame. Implementations generally copy input pixels * Manages a GLSL shader program for processing a frame. Implementations generally copy input pixels
* into an output frame, with changes to pixels specific to the implementation. * into an output frame, with changes to pixels specific to the implementation.
...@@ -38,20 +25,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -38,20 +25,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* *
* <p>All methods in this class must be called on the thread that owns the OpenGL context. * <p>All methods in this class must be called on the thread that owns the OpenGL context.
*/ */
public abstract class SingleFrameGlShaderProgram implements GlShaderProgram { public abstract class SingleFrameGlShaderProgram extends BaseGlShaderProgram {
private final boolean useHdr;
private GlObjectsProvider glObjectsProvider; // TODO(b/275384398): Remove this class as it only wraps the BaseGlShaderProgram.
private InputListener inputListener;
private OutputListener outputListener;
private ErrorListener errorListener;
private Executor errorListenerExecutor;
private int inputWidth;
private int inputHeight;
private @MonotonicNonNull GlTextureInfo outputTexture;
private boolean outputTextureInUse;
private boolean frameProcessingStarted;
/** /**
* Creates a {@code SingleFrameGlShaderProgram} instance. * Creates a {@code SingleFrameGlShaderProgram} instance.
...@@ -60,155 +36,6 @@ public abstract class SingleFrameGlShaderProgram implements GlShaderProgram { ...@@ -60,155 +36,6 @@ public abstract class SingleFrameGlShaderProgram implements GlShaderProgram {
* in linear RGB BT.2020. If {@code false}, colors will be in linear RGB BT.709. * in linear RGB BT.2020. If {@code false}, colors will be in linear RGB BT.709.
*/ */
public SingleFrameGlShaderProgram(boolean useHdr) { public SingleFrameGlShaderProgram(boolean useHdr) {
this.useHdr = useHdr; super(useHdr, /* texturePoolCapacity= */ 1);
glObjectsProvider = GlObjectsProvider.DEFAULT;
inputListener = new InputListener() {};
outputListener = new OutputListener() {};
errorListener = (videoFrameProcessingException) -> {};
errorListenerExecutor = MoreExecutors.directExecutor();
}
/**
* Configures the instance based on the input dimensions.
*
* <p>This method must be called before {@linkplain #drawFrame(int,long) drawing} the first frame
* and before drawing subsequent frames with different input dimensions.
*
* @param inputWidth The input width, in pixels.
* @param inputHeight The input height, in pixels.
* @return The output width and height of frames processed through {@link #drawFrame(int, long)}.
* @throws VideoFrameProcessingException If an error occurs while configuring.
*/
public abstract Size configure(int inputWidth, int inputHeight)
throws VideoFrameProcessingException;
/**
* Draws one frame.
*
* <p>This method may only be called after the shader program has been {@link #configure(int, int)
* configured}. The caller is responsible for focussing the correct render target before calling
* this method.
*
* <p>A minimal implementation should tell OpenGL to use its shader program, bind the shader
* program's vertex attributes and uniforms, and issue a drawing command.
*
* @param inputTexId Identifier of a 2D OpenGL texture containing the input frame.
* @param presentationTimeUs The presentation timestamp of the current frame, in microseconds.
* @throws VideoFrameProcessingException If an error occurs while processing or drawing the frame.
*/
public abstract void drawFrame(int inputTexId, long presentationTimeUs)
throws VideoFrameProcessingException;
@Override
public void setGlObjectsProvider(GlObjectsProvider glObjectsProvider) {
checkState(
!frameProcessingStarted,
"The GlObjectsProvider cannot be set after frame processing has started.");
this.glObjectsProvider = glObjectsProvider;
}
@Override
public final void setInputListener(InputListener inputListener) {
this.inputListener = inputListener;
if (!outputTextureInUse) {
inputListener.onReadyToAcceptInputFrame();
}
}
@Override
public final void setOutputListener(OutputListener outputListener) {
this.outputListener = outputListener;
}
@Override
public final void setErrorListener(Executor errorListenerExecutor, ErrorListener errorListener) {
this.errorListenerExecutor = errorListenerExecutor;
this.errorListener = errorListener;
}
@Override
public final void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) {
checkState(
!outputTextureInUse,
"The shader program does not currently accept input frames. Release prior output frames"
+ " first.");
frameProcessingStarted = true;
try {
if (outputTexture == null
|| inputTexture.width != inputWidth
|| inputTexture.height != inputHeight) {
configureOutputTexture(inputTexture.width, inputTexture.height);
}
outputTextureInUse = true;
GlUtil.focusFramebufferUsingCurrentContext(
outputTexture.fboId, outputTexture.width, outputTexture.height);
GlUtil.clearOutputFrame();
drawFrame(inputTexture.texId, presentationTimeUs);
inputListener.onInputFrameProcessed(inputTexture);
outputListener.onOutputFrameAvailable(outputTexture, presentationTimeUs);
} catch (VideoFrameProcessingException | GlUtil.GlException | RuntimeException e) {
errorListenerExecutor.execute(
() ->
errorListener.onError(
e instanceof VideoFrameProcessingException
? (VideoFrameProcessingException) e
: new VideoFrameProcessingException(e)));
}
}
@EnsuresNonNull("outputTexture")
private void configureOutputTexture(int inputWidth, int inputHeight)
throws GlUtil.GlException, VideoFrameProcessingException {
this.inputWidth = inputWidth;
this.inputHeight = inputHeight;
Size outputSize = configure(inputWidth, inputHeight);
if (outputTexture == null
|| outputSize.getWidth() != outputTexture.width
|| outputSize.getHeight() != outputTexture.height) {
if (outputTexture != null) {
GlUtil.deleteTexture(outputTexture.texId);
GlUtil.deleteFbo(outputTexture.fboId);
}
int outputTexId = GlUtil.createTexture(outputSize.getWidth(), outputSize.getHeight(), useHdr);
outputTexture =
glObjectsProvider.createBuffersForTexture(
outputTexId, outputSize.getWidth(), outputSize.getHeight());
}
}
@Override
public final void releaseOutputFrame(GlTextureInfo outputTexture) {
outputTextureInUse = false;
frameProcessingStarted = true;
inputListener.onReadyToAcceptInputFrame();
}
@Override
public final void signalEndOfCurrentInputStream() {
frameProcessingStarted = true;
outputListener.onCurrentOutputStreamEnded();
}
@Override
@CallSuper
public void flush() {
outputTextureInUse = false;
frameProcessingStarted = true;
inputListener.onFlush();
inputListener.onReadyToAcceptInputFrame();
}
@Override
@CallSuper
public void release() throws VideoFrameProcessingException {
frameProcessingStarted = true;
if (outputTexture != null) {
try {
GlUtil.deleteTexture(outputTexture.texId);
GlUtil.deleteFbo(outputTexture.fboId);
} catch (GlUtil.GlException e) {
throw new VideoFrameProcessingException(e);
}
}
} }
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment