Skip to content

SDK / exoplayer

Go to a project
Commit e5527a8a by hschlueter Committed by Rohit Singh
Options

FrameProcessor: Replace SurfaceInfo.Provider with setter. 

The FinalMatrixTransformationProcessorWrapper ensures that the
surface is only replaced when it is not being rendered to and vice
versa.

PiperOrigin-RevId: 458007639
parent bf86b603
Inline Side-by-side
Showing with 174 additions and 99 deletions
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/GlEffectsFrameProcessorPixelTest.java
...@@ -357,6 +357,16 @@ public final class GlEffectsFrameProcessorPixelTest { ...@@ -357,6 +357,16 @@ public final class GlEffectsFrameProcessorPixelTest {
context, context,
new FrameProcessor.Listener() { new FrameProcessor.Listener() {
@Override @Override
public void onOutputSizeChanged(int width, int height) {
outputImageReader =
ImageReader.newInstance(
width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1);
checkNotNull(glEffectsFrameProcessor)
.setOutputSurfaceInfo(
new SurfaceInfo(outputImageReader.getSurface(), width, height));
}
@Override
public void onFrameProcessingError(FrameProcessingException exception) { public void onFrameProcessingError(FrameProcessingException exception) {
frameProcessingException.set(exception); frameProcessingException.set(exception);
} }
...@@ -368,16 +378,6 @@ public final class GlEffectsFrameProcessorPixelTest { ...@@ -368,16 +378,6 @@ public final class GlEffectsFrameProcessorPixelTest {
}, },
/* streamOffsetUs= */ 0L, /* streamOffsetUs= */ 0L,
effects, effects,
/* outputSurfaceProvider= */ (requestedWidth, requestedHeight) -> {
outputImageReader =
ImageReader.newInstance(
requestedWidth,
requestedHeight,
PixelFormat.RGBA_8888,
/* maxImages= */ 1);
return new SurfaceInfo(
outputImageReader.getSurface(), requestedWidth, requestedHeight);
},
Transformer.DebugViewProvider.NONE, Transformer.DebugViewProvider.NONE,
/* enableExperimentalHdrEditing= */ false)); /* enableExperimentalHdrEditing= */ false));
glEffectsFrameProcessor.setInputFrameInfo( glEffectsFrameProcessor.setInputFrameInfo(
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FinalMatrixTransformationProcessorWrapper.java
...@@ -34,6 +34,7 @@ import androidx.annotation.WorkerThread; ...@@ -34,6 +34,7 @@ import androidx.annotation.WorkerThread;
import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.util.GlUtil; import com.google.android.exoplayer2.util.GlUtil;
import com.google.android.exoplayer2.util.Log; import com.google.android.exoplayer2.util.Log;
import com.google.android.exoplayer2.util.Util;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
...@@ -57,7 +58,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -57,7 +58,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final ImmutableList<GlMatrixTransformation> matrixTransformations; private final ImmutableList<GlMatrixTransformation> matrixTransformations;
private final EGLDisplay eglDisplay; private final EGLDisplay eglDisplay;
private final EGLContext eglContext; private final EGLContext eglContext;
private final SurfaceInfo.Provider outputSurfaceProvider;
private final long streamOffsetUs; private final long streamOffsetUs;
private final Transformer.DebugViewProvider debugViewProvider; private final Transformer.DebugViewProvider debugViewProvider;
private final FrameProcessor.Listener frameProcessorListener; private final FrameProcessor.Listener frameProcessorListener;
...@@ -66,17 +66,23 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -66,17 +66,23 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private int inputWidth; private int inputWidth;
private int inputHeight; private int inputHeight;
@Nullable private MatrixTransformationProcessor matrixTransformationProcessor; @Nullable private MatrixTransformationProcessor matrixTransformationProcessor;
@Nullable private SurfaceInfo outputSurfaceInfo;
@Nullable private EGLSurface outputEglSurface;
@Nullable private SurfaceViewWrapper debugSurfaceViewWrapper; @Nullable private SurfaceViewWrapper debugSurfaceViewWrapper;
private @MonotonicNonNull Listener listener; private @MonotonicNonNull Listener listener;
private @MonotonicNonNull Size outputSizeBeforeSurfaceTransformation;
@GuardedBy("this")
@Nullable
private SurfaceInfo outputSurfaceInfo;
@GuardedBy("this")
@Nullable
private EGLSurface outputEglSurface;
public FinalMatrixTransformationProcessorWrapper( public FinalMatrixTransformationProcessorWrapper(
Context context, Context context,
EGLDisplay eglDisplay, EGLDisplay eglDisplay,
EGLContext eglContext, EGLContext eglContext,
ImmutableList<GlMatrixTransformation> matrixTransformations, ImmutableList<GlMatrixTransformation> matrixTransformations,
SurfaceInfo.Provider outputSurfaceProvider,
long streamOffsetUs, long streamOffsetUs,
FrameProcessor.Listener frameProcessorListener, FrameProcessor.Listener frameProcessorListener,
Transformer.DebugViewProvider debugViewProvider, Transformer.DebugViewProvider debugViewProvider,
...@@ -85,7 +91,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -85,7 +91,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.matrixTransformations = matrixTransformations; this.matrixTransformations = matrixTransformations;
this.eglDisplay = eglDisplay; this.eglDisplay = eglDisplay;
this.eglContext = eglContext; this.eglContext = eglContext;
this.outputSurfaceProvider = outputSurfaceProvider;
this.streamOffsetUs = streamOffsetUs; this.streamOffsetUs = streamOffsetUs;
this.debugViewProvider = debugViewProvider; this.debugViewProvider = debugViewProvider;
this.frameProcessorListener = frameProcessorListener; this.frameProcessorListener = frameProcessorListener;
...@@ -107,6 +112,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -107,6 +112,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@Override @Override
public boolean maybeQueueInputFrame(TextureInfo inputTexture, long presentationTimeUs) { public boolean maybeQueueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
try { try {
synchronized (this) {
if (!ensureConfigured(inputTexture.width, inputTexture.height)) { if (!ensureConfigured(inputTexture.width, inputTexture.height)) {
return false; return false;
} }
...@@ -129,6 +135,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -129,6 +135,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
outputEglSurface, outputEglSurface,
/* presentationTimeNs= */ (presentationTimeUs + streamOffsetUs) * 1000); /* presentationTimeNs= */ (presentationTimeUs + streamOffsetUs) * 1000);
EGL14.eglSwapBuffers(eglDisplay, outputEglSurface); EGL14.eglSwapBuffers(eglDisplay, outputEglSurface);
}
} catch (FrameProcessingException | GlUtil.GlException e) { } catch (FrameProcessingException | GlUtil.GlException e) {
frameProcessorListener.onFrameProcessingError( frameProcessorListener.onFrameProcessingError(
FrameProcessingException.from(e, presentationTimeUs)); FrameProcessingException.from(e, presentationTimeUs));
...@@ -156,24 +163,25 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -156,24 +163,25 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@EnsuresNonNullIf( @EnsuresNonNullIf(
expression = {"outputSurfaceInfo", "outputEglSurface", "matrixTransformationProcessor"}, expression = {"outputSurfaceInfo", "outputEglSurface", "matrixTransformationProcessor"},
result = true) result = true)
private boolean ensureConfigured(int inputWidth, int inputHeight) private synchronized boolean ensureConfigured(int inputWidth, int inputHeight)
throws FrameProcessingException, GlUtil.GlException { throws FrameProcessingException, GlUtil.GlException {
if (inputWidth == this.inputWidth
&& inputHeight == this.inputHeight
&& outputSurfaceInfo != null
&& outputEglSurface != null
&& matrixTransformationProcessor != null) {
return true;
}
if (this.inputWidth != inputWidth
|| this.inputHeight != inputHeight
|| this.outputSizeBeforeSurfaceTransformation == null) {
this.inputWidth = inputWidth; this.inputWidth = inputWidth;
this.inputHeight = inputHeight; this.inputHeight = inputHeight;
Size requestedOutputSize = Size outputSizeBeforeSurfaceTransformation =
MatrixUtils.configureAndGetOutputSize(inputWidth, inputHeight, matrixTransformations); MatrixUtils.configureAndGetOutputSize(inputWidth, inputHeight, matrixTransformations);
@Nullable if (!Util.areEqual(
SurfaceInfo outputSurfaceInfo = this.outputSizeBeforeSurfaceTransformation, outputSizeBeforeSurfaceTransformation)) {
outputSurfaceProvider.getSurfaceInfo( this.outputSizeBeforeSurfaceTransformation = outputSizeBeforeSurfaceTransformation;
requestedOutputSize.getWidth(), requestedOutputSize.getHeight()); frameProcessorListener.onOutputSizeChanged(
outputSizeBeforeSurfaceTransformation.getWidth(),
outputSizeBeforeSurfaceTransformation.getHeight());
}
}
if (outputSurfaceInfo == null) { if (outputSurfaceInfo == null) {
if (matrixTransformationProcessor != null) { if (matrixTransformationProcessor != null) {
matrixTransformationProcessor.release(); matrixTransformationProcessor.release();
...@@ -182,13 +190,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -182,13 +190,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
outputEglSurface = null; outputEglSurface = null;
return false; return false;
} }
if (outputSurfaceInfo == this.outputSurfaceInfo
&& outputEglSurface != null
&& matrixTransformationProcessor != null) {
return true;
}
EGLSurface outputEglSurface; SurfaceInfo outputSurfaceInfo = this.outputSurfaceInfo;
@Nullable EGLSurface outputEglSurface = this.outputEglSurface;
if (outputEglSurface == null) { // This means that outputSurfaceInfo changed.
if (enableExperimentalHdrEditing) { if (enableExperimentalHdrEditing) {
// TODO(b/227624622): Don't assume BT.2020 PQ input/output. // TODO(b/227624622): Don't assume BT.2020 PQ input/output.
outputEglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, outputSurfaceInfo.surface); outputEglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, outputSurfaceInfo.surface);
...@@ -205,9 +210,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -205,9 +210,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
new SurfaceViewWrapper( new SurfaceViewWrapper(
eglDisplay, eglContext, enableExperimentalHdrEditing, debugSurfaceView); eglDisplay, eglContext, enableExperimentalHdrEditing, debugSurfaceView);
} }
if (matrixTransformationProcessor != null) {
matrixTransformationProcessor.release();
matrixTransformationProcessor = null;
}
}
if (matrixTransformationProcessor == null) {
matrixTransformationProcessor = matrixTransformationProcessor =
createMatrixTransformationProcessorForOutputSurface(requestedOutputSize, outputSurfaceInfo); createMatrixTransformationProcessorForOutputSurface(outputSurfaceInfo);
}
this.outputSurfaceInfo = outputSurfaceInfo; this.outputSurfaceInfo = outputSurfaceInfo;
this.outputEglSurface = outputEglSurface; this.outputEglSurface = outputEglSurface;
...@@ -215,7 +227,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -215,7 +227,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
private MatrixTransformationProcessor createMatrixTransformationProcessorForOutputSurface( private MatrixTransformationProcessor createMatrixTransformationProcessorForOutputSurface(
Size requestedOutputSize, SurfaceInfo outputSurfaceInfo) throws FrameProcessingException { SurfaceInfo outputSurfaceInfo) throws FrameProcessingException {
ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder = ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder =
new ImmutableList.Builder<GlMatrixTransformation>().addAll(matrixTransformations); new ImmutableList.Builder<GlMatrixTransformation>().addAll(matrixTransformations);
if (outputSurfaceInfo.orientationDegrees != 0) { if (outputSurfaceInfo.orientationDegrees != 0) {
...@@ -224,12 +236,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -224,12 +236,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
.setRotationDegrees(outputSurfaceInfo.orientationDegrees) .setRotationDegrees(outputSurfaceInfo.orientationDegrees)
.build()); .build());
} }
if (outputSurfaceInfo.width != requestedOutputSize.getWidth()
|| outputSurfaceInfo.height != requestedOutputSize.getHeight()) {
matrixTransformationListBuilder.add( matrixTransformationListBuilder.add(
Presentation.createForWidthAndHeight( Presentation.createForWidthAndHeight(
outputSurfaceInfo.width, outputSurfaceInfo.height, Presentation.LAYOUT_SCALE_TO_FIT)); outputSurfaceInfo.width, outputSurfaceInfo.height, Presentation.LAYOUT_SCALE_TO_FIT));
}
MatrixTransformationProcessor matrixTransformationProcessor = MatrixTransformationProcessor matrixTransformationProcessor =
new MatrixTransformationProcessor(context, matrixTransformationListBuilder.build()); new MatrixTransformationProcessor(context, matrixTransformationListBuilder.build());
...@@ -258,6 +267,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -258,6 +267,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
} }
public synchronized void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) {
if (!Util.areEqual(this.outputSurfaceInfo, outputSurfaceInfo)) {
this.outputSurfaceInfo = outputSurfaceInfo;
this.outputEglSurface = null;
}
}
/** /**
* Wrapper around a {@link SurfaceView} that keeps track of whether the output surface is valid, * Wrapper around a {@link SurfaceView} that keeps track of whether the output surface is valid,
* and makes rendering a no-op if not. * and makes rendering a no-op if not.
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameProcessor.java
...@@ -16,6 +16,7 @@ ...@@ -16,6 +16,7 @@
package com.google.android.exoplayer2.transformer; package com.google.android.exoplayer2.transformer;
import android.view.Surface; import android.view.Surface;
import androidx.annotation.Nullable;
/** Interface for a frame processor that applies changes to individual video frames. */ /** Interface for a frame processor that applies changes to individual video frames. */
/* package */ interface FrameProcessor { /* package */ interface FrameProcessor {
...@@ -27,6 +28,14 @@ import android.view.Surface; ...@@ -27,6 +28,14 @@ import android.view.Surface;
interface Listener { interface Listener {
/** /**
* Called when the output size after applying the final effect changes.
*
* <p>The output size after applying the final effect can differ from the size specified using
* {@link #setOutputSurfaceInfo(SurfaceInfo)}.
*/
void onOutputSizeChanged(int width, int height);
/**
* Called when an exception occurs during asynchronous frame processing. * Called when an exception occurs during asynchronous frame processing.
* *
* <p>If an error occurred, consuming and producing further frames will not work as expected and * <p>If an error occurred, consuming and producing further frames will not work as expected and
...@@ -69,6 +78,23 @@ import android.view.Surface; ...@@ -69,6 +78,23 @@ import android.view.Surface;
int getPendingInputFrameCount(); int getPendingInputFrameCount();
/** /**
* Sets the output surface and supporting information.
*
* <p>The new output {@link SurfaceInfo} is applied from the next output frame rendered onwards.
* If the output {@link SurfaceInfo} is {@code null}, the {@code FrameProcessor} will stop
* rendering and resume rendering pending frames once a non-null {@link SurfaceInfo} is set.
*
* <p>If the dimensions given in {@link SurfaceInfo} do not match the {@linkplain
* Listener#onOutputSizeChanged(int,int) output size after applying the final effect} the frames
* are resized before rendering to the surface and letter/pillar-boxing is applied.
*
* <p>The caller is responsible for tracking the lifecycle of the {@link SurfaceInfo#surface}
* including calling this method with a new surface if it is destroyed. When this method returns,
* the previous output surface is no longer being used and can safely be released by the caller.
*/
void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo);
/**
* Informs the {@code FrameProcessor} that no further input frames should be accepted. * Informs the {@code FrameProcessor} that no further input frames should be accepted.
* *
* @throws IllegalStateException If called more than once. * @throws IllegalStateException If called more than once.
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlEffectsFrameProcessor.java
...@@ -23,6 +23,7 @@ import android.graphics.SurfaceTexture; ...@@ -23,6 +23,7 @@ import android.graphics.SurfaceTexture;
import android.opengl.EGL14; import android.opengl.EGL14;
import android.opengl.EGLContext; import android.opengl.EGLContext;
import android.opengl.EGLDisplay; import android.opengl.EGLDisplay;
import android.util.Pair;
import android.view.Surface; import android.view.Surface;
import androidx.annotation.Nullable; import androidx.annotation.Nullable;
import androidx.annotation.WorkerThread; import androidx.annotation.WorkerThread;
...@@ -51,8 +52,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -51,8 +52,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* @param context A {@link Context}. * @param context A {@link Context}.
* @param listener A {@link Listener}. * @param listener A {@link Listener}.
* @param effects The {@link GlEffect GlEffects} to apply to each frame. * @param effects The {@link GlEffect GlEffects} to apply to each frame.
* @param outputSurfaceProvider A {@link SurfaceInfo.Provider} managing the output {@link
* Surface}.
* @param debugViewProvider A {@link Transformer.DebugViewProvider}. * @param debugViewProvider A {@link Transformer.DebugViewProvider}.
* @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal. * @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal.
* @return A new instance. * @return A new instance.
...@@ -64,7 +63,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -64,7 +63,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
FrameProcessor.Listener listener, FrameProcessor.Listener listener,
long streamOffsetUs, long streamOffsetUs,
List<GlEffect> effects, List<GlEffect> effects,
SurfaceInfo.Provider outputSurfaceProvider,
Transformer.DebugViewProvider debugViewProvider, Transformer.DebugViewProvider debugViewProvider,
boolean enableExperimentalHdrEditing) boolean enableExperimentalHdrEditing)
throws FrameProcessingException { throws FrameProcessingException {
...@@ -79,7 +77,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -79,7 +77,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
listener, listener,
streamOffsetUs, streamOffsetUs,
effects, effects,
outputSurfaceProvider,
debugViewProvider, debugViewProvider,
enableExperimentalHdrEditing, enableExperimentalHdrEditing,
singleThreadExecutorService)); singleThreadExecutorService));
...@@ -108,7 +105,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -108,7 +105,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
FrameProcessor.Listener listener, FrameProcessor.Listener listener,
long streamOffsetUs, long streamOffsetUs,
List<GlEffect> effects, List<GlEffect> effects,
SurfaceInfo.Provider outputSurfaceProvider,
Transformer.DebugViewProvider debugViewProvider, Transformer.DebugViewProvider debugViewProvider,
boolean enableExperimentalHdrEditing, boolean enableExperimentalHdrEditing,
ExecutorService singleThreadExecutorService) ExecutorService singleThreadExecutorService)
...@@ -131,24 +127,31 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -131,24 +127,31 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
GlUtil.focusPlaceholderEglSurface(eglContext, eglDisplay); GlUtil.focusPlaceholderEglSurface(eglContext, eglDisplay);
} }
ImmutableList<GlTextureProcessor> textureProcessors = Pair<ImmutableList<GlTextureProcessor>, FinalMatrixTransformationProcessorWrapper>
textureProcessors =
getGlTextureProcessorsForGlEffects( getGlTextureProcessorsForGlEffects(
context, context,
effects, effects,
eglDisplay, eglDisplay,
eglContext, eglContext,
outputSurfaceProvider,
streamOffsetUs, streamOffsetUs,
listener, listener,
debugViewProvider, debugViewProvider,
enableExperimentalHdrEditing); enableExperimentalHdrEditing);
ImmutableList<GlTextureProcessor> intermediateTextureProcessors = textureProcessors.first;
FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper =
textureProcessors.second;
ExternalTextureProcessor externalTextureProcessor = ExternalTextureProcessor externalTextureProcessor =
new ExternalTextureProcessor(context, enableExperimentalHdrEditing); new ExternalTextureProcessor(context, enableExperimentalHdrEditing);
FrameProcessingTaskExecutor frameProcessingTaskExecutor = FrameProcessingTaskExecutor frameProcessingTaskExecutor =
new FrameProcessingTaskExecutor(singleThreadExecutorService, listener); new FrameProcessingTaskExecutor(singleThreadExecutorService, listener);
chainTextureProcessorsWithListeners( chainTextureProcessorsWithListeners(
externalTextureProcessor, textureProcessors, frameProcessingTaskExecutor, listener); externalTextureProcessor,
intermediateTextureProcessors,
finalTextureProcessorWrapper,
frameProcessingTaskExecutor,
listener);
return new GlEffectsFrameProcessor( return new GlEffectsFrameProcessor(
eglDisplay, eglDisplay,
...@@ -157,7 +160,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -157,7 +160,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
streamOffsetUs, streamOffsetUs,
/* inputExternalTextureId= */ GlUtil.createExternalTexture(), /* inputExternalTextureId= */ GlUtil.createExternalTexture(),
externalTextureProcessor, externalTextureProcessor,
textureProcessors); intermediateTextureProcessors,
finalTextureProcessorWrapper);
} }
/** /**
...@@ -165,16 +169,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -165,16 +169,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* MatrixTransformationProcessor} and converts all other {@link GlEffect} instances to separate * MatrixTransformationProcessor} and converts all other {@link GlEffect} instances to separate
* {@link GlTextureProcessor} instances. * {@link GlTextureProcessor} instances.
* *
* <p>The final {@link GlTextureProcessor} is wrapped in a {@link * @return A {@link Pair} containing a list of {@link GlTextureProcessor} instances to apply in
* FinalMatrixTransformationProcessorWrapper} so that it can write directly to the {@linkplain * the given order and a {@link FinalMatrixTransformationProcessorWrapper} to apply after
* SurfaceInfo.Provider provided output surface}. * them.
*/ */
private static ImmutableList<GlTextureProcessor> getGlTextureProcessorsForGlEffects( private static Pair<ImmutableList<GlTextureProcessor>, FinalMatrixTransformationProcessorWrapper>
getGlTextureProcessorsForGlEffects(
Context context, Context context,
List<GlEffect> effects, List<GlEffect> effects,
EGLDisplay eglDisplay, EGLDisplay eglDisplay,
EGLContext eglContext, EGLContext eglContext,
SurfaceInfo.Provider outputSurfaceProvider,
long streamOffsetUs, long streamOffsetUs,
FrameProcessor.Listener listener, FrameProcessor.Listener listener,
Transformer.DebugViewProvider debugViewProvider, Transformer.DebugViewProvider debugViewProvider,
...@@ -199,18 +203,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -199,18 +203,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
textureProcessorListBuilder.add(effect.toGlTextureProcessor(context)); textureProcessorListBuilder.add(effect.toGlTextureProcessor(context));
} }
textureProcessorListBuilder.add( return Pair.create(
textureProcessorListBuilder.build(),
new FinalMatrixTransformationProcessorWrapper( new FinalMatrixTransformationProcessorWrapper(
context, context,
eglDisplay, eglDisplay,
eglContext, eglContext,
matrixTransformationListBuilder.build(), matrixTransformationListBuilder.build(),
outputSurfaceProvider,
streamOffsetUs, streamOffsetUs,
listener, listener,
debugViewProvider, debugViewProvider,
enableExperimentalHdrEditing)); enableExperimentalHdrEditing));
return textureProcessorListBuilder.build();
} }
/** /**
...@@ -221,21 +224,26 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -221,21 +224,26 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
*/ */
private static void chainTextureProcessorsWithListeners( private static void chainTextureProcessorsWithListeners(
ExternalTextureProcessor externalTextureProcessor, ExternalTextureProcessor externalTextureProcessor,
ImmutableList<GlTextureProcessor> textureProcessors, ImmutableList<GlTextureProcessor> intermediateTextureProcessors,
FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper,
FrameProcessingTaskExecutor frameProcessingTaskExecutor, FrameProcessingTaskExecutor frameProcessingTaskExecutor,
FrameProcessor.Listener listener) { FrameProcessor.Listener listener) {
externalTextureProcessor.setListener( externalTextureProcessor.setListener(
new ChainingGlTextureProcessorListener( new ChainingGlTextureProcessorListener(
/* previousGlTextureProcessor= */ null, /* previousGlTextureProcessor= */ null,
textureProcessors.get(0), /* nextGlTextureProcessor= */ intermediateTextureProcessors.size() > 0
? intermediateTextureProcessors.get(0)
: finalTextureProcessorWrapper,
frameProcessingTaskExecutor, frameProcessingTaskExecutor,
listener)); listener));
GlTextureProcessor previousGlTextureProcessor = externalTextureProcessor; GlTextureProcessor previousGlTextureProcessor = externalTextureProcessor;
for (int i = 0; i < textureProcessors.size(); i++) { for (int i = 0; i < intermediateTextureProcessors.size(); i++) {
GlTextureProcessor glTextureProcessor = textureProcessors.get(i); GlTextureProcessor glTextureProcessor = intermediateTextureProcessors.get(i);
@Nullable @Nullable
GlTextureProcessor nextGlTextureProcessor = GlTextureProcessor nextGlTextureProcessor =
i + 1 < textureProcessors.size() ? textureProcessors.get(i + 1) : null; i + 1 < intermediateTextureProcessors.size()
? intermediateTextureProcessors.get(i + 1)
: finalTextureProcessorWrapper;
glTextureProcessor.setListener( glTextureProcessor.setListener(
new ChainingGlTextureProcessorListener( new ChainingGlTextureProcessorListener(
previousGlTextureProcessor, previousGlTextureProcessor,
...@@ -244,6 +252,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -244,6 +252,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
listener)); listener));
previousGlTextureProcessor = glTextureProcessor; previousGlTextureProcessor = glTextureProcessor;
} }
finalTextureProcessorWrapper.setListener(
new ChainingGlTextureProcessorListener(
previousGlTextureProcessor,
/* nextGlTextureProcessor= */ null,
frameProcessingTaskExecutor,
listener));
} }
private static final String TAG = "GlEffectsFrameProcessor"; private static final String TAG = "GlEffectsFrameProcessor";
...@@ -267,7 +281,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -267,7 +281,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final float[] inputSurfaceTextureTransformMatrix; private final float[] inputSurfaceTextureTransformMatrix;
private final int inputExternalTextureId; private final int inputExternalTextureId;
private final ExternalTextureProcessor inputExternalTextureProcessor; private final ExternalTextureProcessor inputExternalTextureProcessor;
private final ImmutableList<GlTextureProcessor> textureProcessors; private final ImmutableList<GlTextureProcessor> intermediateTextureProcessors;
private final FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper;
private final ConcurrentLinkedQueue<FrameInfo> pendingInputFrames; private final ConcurrentLinkedQueue<FrameInfo> pendingInputFrames;
private @MonotonicNonNull FrameInfo nextInputFrameInfo; private @MonotonicNonNull FrameInfo nextInputFrameInfo;
...@@ -280,8 +295,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -280,8 +295,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
long streamOffsetUs, long streamOffsetUs,
int inputExternalTextureId, int inputExternalTextureId,
ExternalTextureProcessor inputExternalTextureProcessor, ExternalTextureProcessor inputExternalTextureProcessor,
ImmutableList<GlTextureProcessor> textureProcessors) { ImmutableList<GlTextureProcessor> intermediateTextureProcessors,
checkState(!textureProcessors.isEmpty()); FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper) {
this.eglDisplay = eglDisplay; this.eglDisplay = eglDisplay;
this.eglContext = eglContext; this.eglContext = eglContext;
...@@ -289,7 +304,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -289,7 +304,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.streamOffsetUs = streamOffsetUs; this.streamOffsetUs = streamOffsetUs;
this.inputExternalTextureId = inputExternalTextureId; this.inputExternalTextureId = inputExternalTextureId;
this.inputExternalTextureProcessor = inputExternalTextureProcessor; this.inputExternalTextureProcessor = inputExternalTextureProcessor;
this.textureProcessors = textureProcessors; this.intermediateTextureProcessors = intermediateTextureProcessors;
this.finalTextureProcessorWrapper = finalTextureProcessorWrapper;
inputSurfaceTexture = new SurfaceTexture(inputExternalTextureId); inputSurfaceTexture = new SurfaceTexture(inputExternalTextureId);
inputSurface = new Surface(inputSurfaceTexture); inputSurface = new Surface(inputSurfaceTexture);
...@@ -324,6 +340,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -324,6 +340,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
@Override @Override
public void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) {
finalTextureProcessorWrapper.setOutputSurfaceInfo(outputSurfaceInfo);
}
@Override
public void signalEndOfInputStream() { public void signalEndOfInputStream() {
checkState(!inputStreamEnded); checkState(!inputStreamEnded);
inputStreamEnded = true; inputStreamEnded = true;
...@@ -423,9 +444,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -423,9 +444,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private void releaseTextureProcessorsAndDestroyGlContext() private void releaseTextureProcessorsAndDestroyGlContext()
throws GlUtil.GlException, FrameProcessingException { throws GlUtil.GlException, FrameProcessingException {
inputExternalTextureProcessor.release(); inputExternalTextureProcessor.release();
for (int i = 0; i < textureProcessors.size(); i++) { for (int i = 0; i < intermediateTextureProcessors.size(); i++) {
textureProcessors.get(i).release(); intermediateTextureProcessors.get(i).release();
} }
finalTextureProcessorWrapper.release();
GlUtil.destroyEglContext(eglDisplay, eglContext); GlUtil.destroyEglContext(eglDisplay, eglContext);
} }
} }
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SurfaceInfo.java
...@@ -55,16 +55,27 @@ import androidx.annotation.Nullable; ...@@ -55,16 +55,27 @@ import androidx.annotation.Nullable;
this.orientationDegrees = orientationDegrees; this.orientationDegrees = orientationDegrees;
} }
/** A provider for a {@link SurfaceInfo} instance. */ @Override
public interface Provider { public boolean equals(@Nullable Object o) {
/** if (this == o) {
* Provides a {@linkplain SurfaceInfo surface} for the requested dimensions. return true;
* }
* <p>The dimensions given in the provided {@link SurfaceInfo} may differ from the requested if (!(o instanceof SurfaceInfo)) {
* dimensions. It is up to the caller to transform frames from the requested dimensions to the return false;
* provided dimensions before rendering them to the {@link SurfaceInfo#surface}. }
*/ SurfaceInfo that = (SurfaceInfo) o;
@Nullable return width == that.width
SurfaceInfo getSurfaceInfo(int requestedWidth, int requestedHeight); && height == that.height
&& orientationDegrees == that.orientationDegrees
&& surface.equals(that.surface);
}
@Override
public int hashCode() {
int result = surface.hashCode();
result = 31 * result + width;
result = 31 * result + height;
result = 31 * result + orientationDegrees;
return result;
} }
} }
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java
...@@ -95,8 +95,7 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -95,8 +95,7 @@ import org.checkerframework.dataflow.qual.Pure;
inputFormat, inputFormat,
allowedOutputMimeTypes, allowedOutputMimeTypes,
transformationRequest, transformationRequest,
fallbackListener, fallbackListener);
asyncErrorListener);
try { try {
frameProcessor = frameProcessor =
...@@ -104,6 +103,16 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -104,6 +103,16 @@ import org.checkerframework.dataflow.qual.Pure;
context, context,
new FrameProcessor.Listener() { new FrameProcessor.Listener() {
@Override @Override
public void onOutputSizeChanged(int width, int height) {
try {
checkNotNull(frameProcessor)
.setOutputSurfaceInfo(encoderWrapper.getSurfaceInfo(width, height));
} catch (TransformationException exception) {
asyncErrorListener.onTransformationException(exception);
}
}
@Override
public void onFrameProcessingError(FrameProcessingException exception) { public void onFrameProcessingError(FrameProcessingException exception) {
asyncErrorListener.onTransformationException( asyncErrorListener.onTransformationException(
TransformationException.createForFrameProcessingException( TransformationException.createForFrameProcessingException(
...@@ -121,7 +130,6 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -121,7 +130,6 @@ import org.checkerframework.dataflow.qual.Pure;
}, },
streamOffsetUs, streamOffsetUs,
effectsListBuilder.build(), effectsListBuilder.build(),
/* outputSurfaceProvider= */ encoderWrapper,
debugViewProvider, debugViewProvider,
transformationRequest.enableHdrEditing); transformationRequest.enableHdrEditing);
} catch (FrameProcessingException e) { } catch (FrameProcessingException e) {
...@@ -284,14 +292,13 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -284,14 +292,13 @@ import org.checkerframework.dataflow.qual.Pure;
* dimensions, the same encoder is used and the provided dimensions stay fixed. * dimensions, the same encoder is used and the provided dimensions stay fixed.
*/ */
@VisibleForTesting @VisibleForTesting
/* package */ static final class EncoderWrapper implements SurfaceInfo.Provider { /* package */ static final class EncoderWrapper {
private final Codec.EncoderFactory encoderFactory; private final Codec.EncoderFactory encoderFactory;
private final Format inputFormat; private final Format inputFormat;
private final List<String> allowedOutputMimeTypes; private final List<String> allowedOutputMimeTypes;
private final TransformationRequest transformationRequest; private final TransformationRequest transformationRequest;
private final FallbackListener fallbackListener; private final FallbackListener fallbackListener;
private final Transformer.AsyncErrorListener asyncErrorListener;
private @MonotonicNonNull SurfaceInfo encoderSurfaceInfo; private @MonotonicNonNull SurfaceInfo encoderSurfaceInfo;
...@@ -304,20 +311,18 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -304,20 +311,18 @@ import org.checkerframework.dataflow.qual.Pure;
Format inputFormat, Format inputFormat,
List<String> allowedOutputMimeTypes, List<String> allowedOutputMimeTypes,
TransformationRequest transformationRequest, TransformationRequest transformationRequest,
FallbackListener fallbackListener, FallbackListener fallbackListener) {
Transformer.AsyncErrorListener asyncErrorListener) {
this.encoderFactory = encoderFactory; this.encoderFactory = encoderFactory;
this.inputFormat = inputFormat; this.inputFormat = inputFormat;
this.allowedOutputMimeTypes = allowedOutputMimeTypes; this.allowedOutputMimeTypes = allowedOutputMimeTypes;
this.transformationRequest = transformationRequest; this.transformationRequest = transformationRequest;
this.fallbackListener = fallbackListener; this.fallbackListener = fallbackListener;
this.asyncErrorListener = asyncErrorListener;
} }
@Override
@Nullable @Nullable
public SurfaceInfo getSurfaceInfo(int requestedWidth, int requestedHeight) { public SurfaceInfo getSurfaceInfo(int requestedWidth, int requestedHeight)
throws TransformationException {
if (releaseEncoder) { if (releaseEncoder) {
return null; return null;
} }
...@@ -349,13 +354,8 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -349,13 +354,8 @@ import org.checkerframework.dataflow.qual.Pure;
: inputFormat.sampleMimeType) : inputFormat.sampleMimeType)
.build(); .build();
try {
encoder = encoder =
encoderFactory.createForVideoEncoding(requestedEncoderFormat, allowedOutputMimeTypes); encoderFactory.createForVideoEncoding(requestedEncoderFormat, allowedOutputMimeTypes);
} catch (TransformationException e) {
asyncErrorListener.onTransformationException(e);
return null;
}
Format encoderSupportedFormat = encoder.getConfigurationFormat(); Format encoderSupportedFormat = encoder.getConfigurationFormat();
fallbackListener.onTransformationRequestFinalized( fallbackListener.onTransformationRequestFinalized(
createFallbackTransformationRequest( createFallbackTransformationRequest(
......
library/transformer/src/test/java/com/google/android/exoplayer2/transformer/VideoEncoderWrapperTest.java
...@@ -50,8 +50,7 @@ public final class VideoEncoderWrapperTest { ...@@ -50,8 +50,7 @@ public final class VideoEncoderWrapperTest {
/* inputFormat= */ new Format.Builder().build(), /* inputFormat= */ new Format.Builder().build(),
/* allowedOutputMimeTypes= */ ImmutableList.of(), /* allowedOutputMimeTypes= */ ImmutableList.of(),
emptyTransformationRequest, emptyTransformationRequest,
fallbackListener, fallbackListener);
mock(Transformer.AsyncErrorListener.class));
@Before @Before
public void registerTrack() { public void registerTrack() {
...@@ -59,7 +58,7 @@ public final class VideoEncoderWrapperTest { ...@@ -59,7 +58,7 @@ public final class VideoEncoderWrapperTest {
} }
@Test @Test
public void getSurfaceInfo_landscape_leavesOrientationUnchanged() { public void getSurfaceInfo_landscape_leavesOrientationUnchanged() throws Exception {
int inputWidth = 200; int inputWidth = 200;
int inputHeight = 150; int inputHeight = 150;
...@@ -71,7 +70,7 @@ public final class VideoEncoderWrapperTest { ...@@ -71,7 +70,7 @@ public final class VideoEncoderWrapperTest {
} }
@Test @Test
public void getSurfaceInfo_square_leavesOrientationUnchanged() { public void getSurfaceInfo_square_leavesOrientationUnchanged() throws Exception {
int inputWidth = 150; int inputWidth = 150;
int inputHeight = 150; int inputHeight = 150;
...@@ -83,7 +82,7 @@ public final class VideoEncoderWrapperTest { ...@@ -83,7 +82,7 @@ public final class VideoEncoderWrapperTest {
} }
@Test @Test
public void getSurfaceInfo_portrait_flipsOrientation() { public void getSurfaceInfo_portrait_flipsOrientation() throws Exception {
int inputWidth = 150; int inputWidth = 150;
int inputHeight = 200; int inputHeight = 200;
...@@ -95,7 +94,8 @@ public final class VideoEncoderWrapperTest { ...@@ -95,7 +94,8 @@ public final class VideoEncoderWrapperTest {
} }
@Test @Test
public void getSurfaceInfo_withEncoderFallback_usesFallbackResolution() { public void getSurfaceInfo_withEncoderFallback_usesFallbackResolution()
throws TransformationException {
int inputWidth = 200; int inputWidth = 200;
int inputHeight = 150; int inputHeight = 150;
int fallbackWidth = 100; int fallbackWidth = 100;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment