Skip to content

SDK / exoplayer

Go to a project
Commit e6c8136a by claincly Committed by Marc Baechinger
Options

Add an input switcher to switch between input types. 

Also make FinalShaderProgramWrapper always receive internal texture.

This means it does not sample from a input texture, and its input color is
always linear, hence the input type does not matter.

PiperOrigin-RevId: 527869045
parent ca4928cf
Inline Side-by-side
Showing with 366 additions and 156 deletions
library/effect/src/main/java/com/google/android/exoplayer2/effect/BitmapTextureManager.java
......@@ -36,7 +36,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
*
* <p>Public methods in this class can be called from any thread.
*/
/* package */ final class BitmapTextureManager implements InputHandler {
/* package */ final class BitmapTextureManager implements TextureManager {
private final GlShaderProgram shaderProgram;
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
// The queue holds all bitmaps with one or more frames pending to be sent downstream.
......
library/effect/src/main/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessor.java
......@@ -249,11 +249,15 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
private final EGLDisplay eglDisplay;
private final EGLContext eglContext;
private final InputSwitcher inputSwitcher;
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
private final InputHandler inputHandler;
// TODO(b/274109008) Use InputSwither to interact with texture manager.
// Owned and released by inputSwitcher.
private final TextureManager textureManager;
private final boolean renderFramesAutomatically;
private final FinalShaderProgramWrapper finalShaderProgramWrapper;
private final ImmutableList<GlShaderProgram> allShaderPrograms;
// Shader programs that apply Effects.
private final ImmutableList<GlShaderProgram> effectsShaderPrograms;
// A queue of input streams that have not been fully processed identified by their input types.
private final Queue<@InputType Integer> unprocessedInputStreams;
......@@ -266,41 +270,23 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
private DefaultVideoFrameProcessor(
EGLDisplay eglDisplay,
EGLContext eglContext,
InputSwitcher inputSwitcher,
@InputType int inputType,
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
ImmutableList<GlShaderProgram> shaderPrograms,
boolean renderFramesAutomatically)
throws VideoFrameProcessingException {
ImmutableList<GlShaderProgram> effectsShaderPrograms,
boolean renderFramesAutomatically) {
this.eglDisplay = eglDisplay;
this.eglContext = eglContext;
this.inputSwitcher = inputSwitcher;
this.videoFrameProcessingTaskExecutor = videoFrameProcessingTaskExecutor;
this.renderFramesAutomatically = renderFramesAutomatically;
this.unprocessedInputStreams = new ConcurrentLinkedQueue<>();
checkState(!shaderPrograms.isEmpty());
checkState(getLast(shaderPrograms) instanceof FinalShaderProgramWrapper);
GlShaderProgram inputShaderProgram = shaderPrograms.get(0);
switch (inputType) {
case VideoFrameProcessor.INPUT_TYPE_SURFACE:
checkState(inputShaderProgram instanceof ExternalShaderProgram);
inputHandler =
new ExternalTextureManager(
(ExternalShaderProgram) inputShaderProgram, videoFrameProcessingTaskExecutor);
break;
case VideoFrameProcessor.INPUT_TYPE_BITMAP:
inputHandler =
new BitmapTextureManager(inputShaderProgram, videoFrameProcessingTaskExecutor);
break;
case VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID: // fall through
default:
throw new VideoFrameProcessingException("Input type not supported yet");
}
inputShaderProgram.setInputListener(inputHandler);
checkState(!effectsShaderPrograms.isEmpty());
checkState(getLast(effectsShaderPrograms) instanceof FinalShaderProgramWrapper);
finalShaderProgramWrapper = (FinalShaderProgramWrapper) getLast(shaderPrograms);
textureManager = inputSwitcher.switchToInput(inputType);
finalShaderProgramWrapper = (FinalShaderProgramWrapper) getLast(effectsShaderPrograms);
finalShaderProgramWrapper.setOnInputStreamProcessedListener(
() -> {
@InputType int currentInputType = unprocessedInputStreams.remove();
......@@ -317,7 +303,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
}
return inputStreamEnded && unprocessedInputStreams.isEmpty();
});
allShaderPrograms = shaderPrograms;
this.effectsShaderPrograms = effectsShaderPrograms;
}
/** Returns the task executor that runs video frame processing tasks. */
......@@ -342,7 +328,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
* @param height The default height for input buffers, in pixels.
*/
public void setInputDefaultBufferSize(int width, int height) {
inputHandler.setDefaultBufferSize(width, height);
textureManager.setDefaultBufferSize(width, height);
}
@Override
......@@ -350,7 +336,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
checkState(
hasRefreshedNextInputFrameInfo,
"setInputFrameInfo must be called before queueing another bitmap");
inputHandler.queueInputBitmap(
textureManager.queueInputBitmap(
inputBitmap,
durationUs,
checkNotNull(nextInputFrameInfo).offsetToAddUs,
......@@ -361,13 +347,13 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
@Override
public Surface getInputSurface() {
return inputHandler.getInputSurface();
return textureManager.getInputSurface();
}
@Override
public void registerInputStream(@InputType int inputType) {
if (!unprocessedInputStreams.isEmpty()) {
inputHandler.signalEndOfCurrentInputStream();
textureManager.signalEndOfCurrentInputStream();
// Wait until the current video is processed before continuing to the next input.
if (checkNotNull(unprocessedInputStreams.peek()) == INPUT_TYPE_SURFACE) {
latch = new CountDownLatch(1);
......@@ -394,13 +380,13 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
checkStateNotNull(
nextInputFrameInfo, "setInputFrameInfo must be called before registering input frames");
inputHandler.registerInputFrame(nextInputFrameInfo);
textureManager.registerInputFrame(nextInputFrameInfo);
hasRefreshedNextInputFrameInfo = false;
}
@Override
public int getPendingInputFrameCount() {
return inputHandler.getPendingFrameCount();
return textureManager.getPendingFrameCount();
}
@Override
......@@ -421,8 +407,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
public void signalEndOfInput() {
checkState(!inputStreamEnded);
inputStreamEnded = true;
inputHandler.signalEndOfCurrentInputStream();
inputHandler.signalEndOfInput();
textureManager.signalEndOfCurrentInputStream();
inputSwitcher.signalEndOfInput();
}
@Override
......@@ -430,10 +416,10 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
try {
videoFrameProcessingTaskExecutor.flush();
CountDownLatch latch = new CountDownLatch(1);
inputHandler.setOnFlushCompleteListener(latch::countDown);
textureManager.setOnFlushCompleteListener(latch::countDown);
videoFrameProcessingTaskExecutor.submit(finalShaderProgramWrapper::flush);
latch.await();
inputHandler.setOnFlushCompleteListener(null);
textureManager.setOnFlushCompleteListener(null);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
......@@ -448,7 +434,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
Thread.currentThread().interrupt();
throw new IllegalStateException(unexpected);
}
inputHandler.release();
}
/**
......@@ -524,35 +509,54 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
throw new VideoFrameProcessingException("BT.2020 PQ OpenGL output isn't supported.");
}
}
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor =
new VideoFrameProcessingTaskExecutor(singleThreadExecutorService, listener);
ColorInfo linearColorInfo =
outputColorInfo
.buildUpon()
.setColorTransfer(C.COLOR_TRANSFER_LINEAR)
.setHdrStaticInfo(null)
.build();
InputSwitcher inputSwitcher =
new InputSwitcher(
context,
inputColorInfo,
/* outputColorInfo= */ linearColorInfo,
glObjectsProvider,
videoFrameProcessingTaskExecutor,
enableColorTransfers);
ImmutableList<GlShaderProgram> shaderPrograms =
ImmutableList<GlShaderProgram> effectsShaderPrograms =
getGlShaderProgramsForGlEffects(
context,
effects,
eglDisplay,
eglContext,
debugViewProvider,
inputColorInfo,
/* inputColorInfo= */ linearColorInfo,
outputColorInfo,
enableColorTransfers,
inputType,
renderFramesAutomatically,
executor,
listener,
glObjectsProvider,
textureOutputListener);
setGlObjectProviderOnShaderPrograms(shaderPrograms, glObjectsProvider);
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor =
new VideoFrameProcessingTaskExecutor(singleThreadExecutorService, listener);
// TODO(b/274109008): Register both image and video input.
inputSwitcher.registerInput(inputType);
inputSwitcher.setDownstreamShaderProgram(effectsShaderPrograms.get(0));
setGlObjectProviderOnShaderPrograms(effectsShaderPrograms, glObjectsProvider);
chainShaderProgramsWithListeners(
shaderPrograms, videoFrameProcessingTaskExecutor, listener, executor);
effectsShaderPrograms, videoFrameProcessingTaskExecutor, listener, executor);
return new DefaultVideoFrameProcessor(
eglDisplay,
eglContext,
inputSwitcher,
inputType,
videoFrameProcessingTaskExecutor,
shaderPrograms,
effectsShaderPrograms,
renderFramesAutomatically);
}
......@@ -564,8 +568,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
* <p>All {@link Effect} instances must be {@link GlEffect} instances.
*
* @return A non-empty list of {@link GlShaderProgram} instances to apply in the given order. The
* first is an {@link ExternalShaderProgram} and the last is a {@link
* FinalShaderProgramWrapper}.
* last is a {@link FinalShaderProgramWrapper}.
*/
private static ImmutableList<GlShaderProgram> getGlShaderProgramsForGlEffects(
Context context,
......@@ -576,7 +579,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
boolean enableColorTransfers,
@InputType int inputType,
boolean renderFramesAutomatically,
Executor executor,
Listener listener,
......@@ -587,13 +589,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder =
new ImmutableList.Builder<>();
ImmutableList.Builder<RgbMatrix> rgbMatrixListBuilder = new ImmutableList.Builder<>();
boolean sampleFromInputTexture = true;
ColorInfo linearColorInfo =
outputColorInfo
.buildUpon()
.setColorTransfer(C.COLOR_TRANSFER_LINEAR)
.setHdrStaticInfo(null)
.build();
for (int i = 0; i < effects.size(); i++) {
Effect effect = effects.get(i);
checkArgument(
......@@ -615,38 +610,13 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
matrixTransformationListBuilder.build();
ImmutableList<RgbMatrix> rgbMatrices = rgbMatrixListBuilder.build();
boolean isOutputTransferHdr = ColorInfo.isTransferHdr(outputColorInfo);
if (!matrixTransformations.isEmpty() || !rgbMatrices.isEmpty() || sampleFromInputTexture) {
DefaultShaderProgram defaultShaderProgram;
if (sampleFromInputTexture) {
if (inputType == INPUT_TYPE_SURFACE) {
defaultShaderProgram =
DefaultShaderProgram.createWithExternalSampler(
context,
matrixTransformations,
rgbMatrices,
inputColorInfo,
linearColorInfo,
enableColorTransfers);
} else {
defaultShaderProgram =
DefaultShaderProgram.createWithInternalSampler(
context,
matrixTransformations,
rgbMatrices,
inputColorInfo,
linearColorInfo,
enableColorTransfers,
inputType);
}
} else {
defaultShaderProgram =
DefaultShaderProgram.create(
context, matrixTransformations, rgbMatrices, isOutputTransferHdr);
}
if (!matrixTransformations.isEmpty() || !rgbMatrices.isEmpty()) {
DefaultShaderProgram defaultShaderProgram =
DefaultShaderProgram.create(
context, matrixTransformations, rgbMatrices, isOutputTransferHdr);
shaderProgramListBuilder.add(defaultShaderProgram);
matrixTransformationListBuilder = new ImmutableList.Builder<>();
rgbMatrixListBuilder = new ImmutableList.Builder<>();
sampleFromInputTexture = false;
}
shaderProgramListBuilder.add(glEffect.toGlShaderProgram(context, isOutputTransferHdr));
}
......@@ -659,11 +629,9 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
matrixTransformationListBuilder.build(),
rgbMatrixListBuilder.build(),
debugViewProvider,
/* inputColorInfo= */ sampleFromInputTexture ? inputColorInfo : linearColorInfo,
inputColorInfo,
outputColorInfo,
enableColorTransfers,
sampleFromInputTexture,
inputType,
renderFramesAutomatically,
executor,
listener,
......@@ -710,12 +678,13 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
*/
private void releaseShaderProgramsAndDestroyGlContext() {
try {
for (int i = 0; i < allShaderPrograms.size(); i++) {
try {
allShaderPrograms.get(i).release();
} catch (Exception e) {
Log.e(TAG, "Error releasing shader program", e);
try {
inputSwitcher.release();
for (int i = 0; i < effectsShaderPrograms.size(); i++) {
effectsShaderPrograms.get(i).release();
}
} catch (Exception e) {
Log.e(TAG, "Error releasing shader program", e);
}
} finally {
try {
......
library/effect/src/main/java/com/google/android/exoplayer2/effect/ExternalTextureManager.java
......@@ -15,6 +15,7 @@
*/
package com.google.android.exoplayer2.effect;
import static com.google.android.exoplayer2.util.Assertions.checkState;
import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
import android.graphics.Bitmap;
......@@ -35,7 +36,7 @@ import java.util.concurrent.atomic.AtomicInteger;
* Forwards externally produced frames that become available via a {@link SurfaceTexture} to an
* {@link ExternalShaderProgram} for consumption.
*/
/* package */ final class ExternalTextureManager implements InputHandler {
/* package */ final class ExternalTextureManager implements TextureManager {
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
private final ExternalShaderProgram externalShaderProgram;
......@@ -167,6 +168,7 @@ import java.util.concurrent.atomic.AtomicInteger;
*/
@Override
public void registerInputFrame(FrameInfo frame) {
checkState(!inputStreamEnded);
pendingFrames.add(frame);
}
......
library/effect/src/main/java/com/google/android/exoplayer2/effect/FinalShaderProgramWrapper.java
......@@ -17,7 +17,6 @@ package com.google.android.exoplayer2.effect;
import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import static com.google.android.exoplayer2.util.Assertions.checkState;
import static com.google.android.exoplayer2.util.VideoFrameProcessor.INPUT_TYPE_SURFACE;
import android.content.Context;
import android.opengl.EGL14;
......@@ -63,7 +62,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* <p>This wrapper is used for the final {@link DefaultShaderProgram} instance in the chain of
* {@link DefaultShaderProgram} instances used by {@link VideoFrameProcessor}.
*/
/* package */ final class FinalShaderProgramWrapper implements ExternalShaderProgram {
/* package */ final class FinalShaderProgramWrapper implements GlShaderProgram {
/** Listener interface for the current input stream ending. */
interface OnInputStreamProcessedListener {
......@@ -82,15 +81,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final EGLDisplay eglDisplay;
private final EGLContext eglContext;
private final DebugViewProvider debugViewProvider;
private final boolean sampleFromInputTexture;
private final @VideoFrameProcessor.InputType int inputType;
private final ColorInfo inputColorInfo;
private final ColorInfo outputColorInfo;
private final boolean enableColorTransfers;
private final boolean renderFramesAutomatically;
private final Executor videoFrameProcessorListenerExecutor;
private final VideoFrameProcessor.Listener videoFrameProcessorListener;
private final float[] textureTransformMatrix;
private final Queue<Pair<GlTextureInfo, Long>> availableFrames;
@Nullable private final DefaultVideoFrameProcessor.TextureOutputListener textureOutputListener;
......@@ -127,8 +122,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
boolean enableColorTransfers,
boolean sampleFromInputTexture,
@VideoFrameProcessor.InputType int inputType,
boolean renderFramesAutomatically,
Executor videoFrameProcessorListenerExecutor,
VideoFrameProcessor.Listener videoFrameProcessorListener,
......@@ -140,9 +133,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.eglDisplay = eglDisplay;
this.eglContext = eglContext;
this.debugViewProvider = debugViewProvider;
this.sampleFromInputTexture = sampleFromInputTexture;
this.inputType = inputType;
this.inputColorInfo = inputColorInfo;
this.outputColorInfo = outputColorInfo;
this.enableColorTransfers = enableColorTransfers;
this.renderFramesAutomatically = renderFramesAutomatically;
......@@ -151,7 +141,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.glObjectsProvider = glObjectsProvider;
this.textureOutputListener = textureOutputListener;
textureTransformMatrix = GlUtil.create4x4IdentityMatrix();
inputListener = new InputListener() {};
availableFrames = new ConcurrentLinkedQueue<>();
}
......@@ -241,20 +230,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
@Override
public void setTextureTransformMatrix(float[] textureTransformMatrix) {
System.arraycopy(
/* src= */ textureTransformMatrix,
/* srcPos= */ 0,
/* dest= */ this.textureTransformMatrix,
/* destPost= */ 0,
/* length= */ textureTransformMatrix.length);
if (defaultShaderProgram != null) {
defaultShaderProgram.setTextureTransformMatrix(textureTransformMatrix);
}
}
@Override
public synchronized void release() throws VideoFrameProcessingException {
if (defaultShaderProgram != null) {
defaultShaderProgram.release();
......@@ -479,38 +454,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
DefaultShaderProgram defaultShaderProgram;
ImmutableList<GlMatrixTransformation> expandedMatrixTransformations =
matrixTransformationListBuilder.build();
if (sampleFromInputTexture) {
if (inputType == INPUT_TYPE_SURFACE) {
defaultShaderProgram =
DefaultShaderProgram.createWithExternalSampler(
context,
expandedMatrixTransformations,
rgbMatrices,
inputColorInfo,
outputColorInfo,
enableColorTransfers);
} else {
defaultShaderProgram =
DefaultShaderProgram.createWithInternalSampler(
context,
expandedMatrixTransformations,
rgbMatrices,
inputColorInfo,
outputColorInfo,
enableColorTransfers,
inputType);
}
} else {
defaultShaderProgram =
DefaultShaderProgram.createApplyingOetf(
context,
expandedMatrixTransformations,
rgbMatrices,
outputColorInfo,
enableColorTransfers);
}
defaultShaderProgram =
DefaultShaderProgram.createApplyingOetf(
context,
expandedMatrixTransformations,
rgbMatrices,
outputColorInfo,
enableColorTransfers);
defaultShaderProgram.setTextureTransformMatrix(textureTransformMatrix);
Size outputSize = defaultShaderProgram.configure(inputWidth, inputHeight);
if (outputSurfaceInfo != null) {
SurfaceInfo outputSurfaceInfo = checkNotNull(this.outputSurfaceInfo);
......
library/effect/src/main/java/com/google/android/exoplayer2/effect/InputSwitcher.java 0 → 100644
/*
* Copyright 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.google.android.exoplayer2.effect;
import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import static com.google.android.exoplayer2.util.Assertions.checkState;
import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
import android.content.Context;
import android.util.SparseArray;
import androidx.annotation.Nullable;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.util.GlObjectsProvider;
import com.google.android.exoplayer2.util.GlTextureInfo;
import com.google.android.exoplayer2.util.VideoFrameProcessingException;
import com.google.android.exoplayer2.util.VideoFrameProcessor;
import com.google.android.exoplayer2.video.ColorInfo;
import com.google.common.collect.ImmutableList;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* A switcher to switch between {@linkplain TextureManager texture managers} of different
* {@linkplain VideoFrameProcessor.InputType input types}.
*/
/* package */ final class InputSwitcher {
private final Context context;
private final ColorInfo inputColorInfo;
private final ColorInfo outputColorInfo;
private final GlObjectsProvider glObjectsProvider;
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
private final SparseArray<Input> inputs;
private final boolean enableColorTransfers;
private @MonotonicNonNull GlShaderProgram downstreamShaderProgram;
private boolean inputEnded;
private int activeInputType;
public InputSwitcher(
Context context,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
GlObjectsProvider glObjectsProvider,
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
boolean enableColorTransfers) {
this.context = context;
this.inputColorInfo = inputColorInfo;
this.outputColorInfo = outputColorInfo;
this.glObjectsProvider = glObjectsProvider;
this.videoFrameProcessingTaskExecutor = videoFrameProcessingTaskExecutor;
this.inputs = new SparseArray<>();
this.enableColorTransfers = enableColorTransfers;
activeInputType = C.INDEX_UNSET;
}
/**
* Registers for a new {@link VideoFrameProcessor.InputType input}.
*
* <p>Can be called multiple times on the same {@link VideoFrameProcessor.InputType inputType},
* with the new inputs overwriting the old ones. For example, a new instance of {@link
* ExternalTextureManager} is created following each call to this method with {@link
* VideoFrameProcessor#INPUT_TYPE_SURFACE}. Effectively, the {@code inputSwitcher} keeps exactly
* one {@link TextureManager} per {@linkplain VideoFrameProcessor.InputType input type}.
*
* <p>Creates an {@link TextureManager} and an appropriate {@linkplain DefaultShaderProgram
* sampler} to sample from the input.
*/
public void registerInput(@VideoFrameProcessor.InputType int inputType)
throws VideoFrameProcessingException {
// TODO(b/274109008): Investigate lazy instantiating the texture managers.
DefaultShaderProgram samplingShaderProgram;
TextureManager textureManager;
// TODO(b/274109008): Refactor DefaultShaderProgram to create a class just for sampling.
switch (inputType) {
case VideoFrameProcessor.INPUT_TYPE_SURFACE:
samplingShaderProgram =
DefaultShaderProgram.createWithExternalSampler(
context,
/* matrixTransformations= */ ImmutableList.of(),
/* rgbMatrices= */ ImmutableList.of(),
inputColorInfo,
outputColorInfo,
enableColorTransfers);
samplingShaderProgram.setGlObjectsProvider(glObjectsProvider);
textureManager =
new ExternalTextureManager(samplingShaderProgram, videoFrameProcessingTaskExecutor);
inputs.put(inputType, new Input(textureManager, samplingShaderProgram));
break;
case VideoFrameProcessor.INPUT_TYPE_BITMAP:
samplingShaderProgram =
DefaultShaderProgram.createWithInternalSampler(
context,
/* matrixTransformations= */ ImmutableList.of(),
/* rgbMatrices= */ ImmutableList.of(),
inputColorInfo,
outputColorInfo,
enableColorTransfers,
inputType);
samplingShaderProgram.setGlObjectsProvider(glObjectsProvider);
textureManager =
new BitmapTextureManager(samplingShaderProgram, videoFrameProcessingTaskExecutor);
inputs.put(inputType, new Input(textureManager, samplingShaderProgram));
break;
case VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID: // fall through
default:
throw new VideoFrameProcessingException("Unsupported input type " + inputType);
}
}
public void setDownstreamShaderProgram(GlShaderProgram downstreamShaderProgram) {
this.downstreamShaderProgram = downstreamShaderProgram;
for (int i = 0; i < inputs.size(); i++) {
@VideoFrameProcessor.InputType int inputType = inputs.keyAt(i);
Input input = inputs.get(inputType);
input.setChainingListener(
new GatedChainingListenerWrapper(
input.samplingGlShaderProgram,
this.downstreamShaderProgram,
videoFrameProcessingTaskExecutor));
}
}
/**
* Switches to a new source of input.
*
* <p>Blocks until the current input stream is processed.
*
* <p>Must be called after the corresponding {@code newInputType} is {@linkplain #registerInput
* registered}.
*
* @param newInputType The new {@link VideoFrameProcessor.InputType} to switch to.
* @return The {@link TextureManager} associated with the {@code newInputType}.
*/
public TextureManager switchToInput(@VideoFrameProcessor.InputType int newInputType) {
checkStateNotNull(downstreamShaderProgram);
checkState(inputs.indexOfKey(newInputType) >= 0, "Input type not registered: " + newInputType);
if (newInputType == activeInputType) {
return inputs.get(activeInputType).textureManager;
}
@Nullable TextureManager activeTextureManager = null;
for (int i = 0; i < inputs.size(); i++) {
@VideoFrameProcessor.InputType int inputType = inputs.keyAt(i);
Input input = inputs.get(inputType);
if (inputType == newInputType) {
input.setActive(true);
downstreamShaderProgram.setInputListener(checkNotNull(input.gatedChainingListenerWrapper));
activeTextureManager = input.textureManager;
} else {
input.setActive(false);
}
}
activeInputType = newInputType;
return checkNotNull(activeTextureManager);
}
/** Signals end of input to all {@linkplain #registerInput registered inputs}. */
public void signalEndOfInput() {
checkState(!inputEnded);
inputEnded = true;
for (int i = 0; i < inputs.size(); i++) {
@VideoFrameProcessor.InputType int inputType = inputs.keyAt(i);
inputs.get(inputType).signalEndOfInput();
}
}
/** Releases the resources. */
public void release() throws VideoFrameProcessingException {
for (int i = 0; i < inputs.size(); i++) {
inputs.get(inputs.keyAt(i)).release();
}
}
/**
* Wraps a {@link TextureManager} and an appropriate {@linkplain GlShaderProgram sampling shader
* program}.
*
* <p>The output is always an internal GL texture.
*/
private static final class Input {
public final TextureManager textureManager;
public final GlShaderProgram samplingGlShaderProgram;
private @MonotonicNonNull GatedChainingListenerWrapper gatedChainingListenerWrapper;
public Input(TextureManager textureManager, GlShaderProgram samplingGlShaderProgram) {
this.textureManager = textureManager;
this.samplingGlShaderProgram = samplingGlShaderProgram;
samplingGlShaderProgram.setInputListener(textureManager);
}
public void setChainingListener(GatedChainingListenerWrapper gatedChainingListenerWrapper) {
this.gatedChainingListenerWrapper = gatedChainingListenerWrapper;
samplingGlShaderProgram.setOutputListener(gatedChainingListenerWrapper);
}
public void setActive(boolean active) {
checkStateNotNull(gatedChainingListenerWrapper);
gatedChainingListenerWrapper.setActive(active);
}
public void signalEndOfInput() {
textureManager.signalEndOfInput();
}
public void release() throws VideoFrameProcessingException {
textureManager.release();
samplingGlShaderProgram.release();
}
}
/**
* Wraps a {@link ChainingGlShaderProgramListener}, with the ability to turn off the event
* listening.
*/
private static final class GatedChainingListenerWrapper
implements GlShaderProgram.OutputListener, GlShaderProgram.InputListener {
private final ChainingGlShaderProgramListener chainingGlShaderProgramListener;
private boolean isActive = false;
public GatedChainingListenerWrapper(
GlShaderProgram producingGlShaderProgram,
GlShaderProgram consumingGlShaderProgram,
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor) {
this.chainingGlShaderProgramListener =
new ChainingGlShaderProgramListener(
producingGlShaderProgram, consumingGlShaderProgram, videoFrameProcessingTaskExecutor);
}
@Override
public void onReadyToAcceptInputFrame() {
if (isActive) {
chainingGlShaderProgramListener.onReadyToAcceptInputFrame();
}
}
@Override
public void onInputFrameProcessed(GlTextureInfo inputTexture) {
if (isActive) {
chainingGlShaderProgramListener.onInputFrameProcessed(inputTexture);
}
}
@Override
public synchronized void onFlush() {
if (isActive) {
chainingGlShaderProgramListener.onFlush();
}
}
@Override
public synchronized void onOutputFrameAvailable(
GlTextureInfo outputTexture, long presentationTimeUs) {
if (isActive) {
chainingGlShaderProgramListener.onOutputFrameAvailable(outputTexture, presentationTimeUs);
}
}
@Override
public synchronized void onCurrentOutputStreamEnded() {
if (isActive) {
chainingGlShaderProgramListener.onCurrentOutputStreamEnded();
}
}
public void setActive(boolean isActive) {
this.isActive = isActive;
}
}
}
library/effect/src/main/java/com/google/android/exoplayer2/effect/InputHandler.java → library/effect/src/main/java/com/google/android/exoplayer2/effect/TextureManager.java
......@@ -25,7 +25,7 @@ import com.google.android.exoplayer2.util.FrameInfo;
import com.google.android.exoplayer2.util.VideoFrameProcessor;
/** A component that handles {@code DefaultVideoFrameProcessor}'s input. */
/* package */ interface InputHandler extends GlShaderProgram.InputListener {
/* package */ interface TextureManager extends GlShaderProgram.InputListener {
/**
* See {@link DefaultVideoFrameProcessor#setInputDefaultBufferSize}.
......@@ -61,7 +61,7 @@ import com.google.android.exoplayer2.util.VideoFrameProcessor;
throw new UnsupportedOperationException();
}
/** Informs the {@code InputHandler} that a frame will be queued. */
/** Informs the {@code TextureManager} that a frame will be queued. */
default void registerInputFrame(FrameInfo frameInfo) {
throw new UnsupportedOperationException();
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment