Skip to content

SDK / exoplayer

Go to a project
Commit 4e63bdf3 by hschlueter Committed by Ian Baker
Options

Move OpenGL setup to FrameProcessorChain#configure(). 

The factory method is replaced by a public constructor and
configure() method which configures the input/output surfaces
and handles the OpenGL setup.

This is a prerequisite for removing the responsibility of the
caller to configureSizes() before creating the chain in a follow-up.

PiperOrigin-RevId: 437028882
parent e18dfd44
Inline Side-by-side
Showing with 233 additions and 308 deletions
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameProcessorChainPixelTest.java
...@@ -251,31 +251,27 @@ public final class FrameProcessorChainPixelTest { ...@@ -251,31 +251,27 @@ public final class FrameProcessorChainPixelTest {
List<Size> sizes = List<Size> sizes =
FrameProcessorChain.configureSizes(inputWidth, inputHeight, frameProcessorsList); FrameProcessorChain.configureSizes(inputWidth, inputHeight, frameProcessorsList);
assertThat(sizes).isNotEmpty(); assertThat(sizes).isNotEmpty();
int outputWidth = Iterables.getLast(sizes).getWidth();
int outputHeight = Iterables.getLast(sizes).getHeight();
outputImageReader = outputImageReader =
ImageReader.newInstance( ImageReader.newInstance(
Iterables.getLast(sizes).getWidth(), outputWidth, outputHeight, PixelFormat.RGBA_8888, /* maxImages= */ 1);
Iterables.getLast(sizes).getHeight(),
PixelFormat.RGBA_8888,
/* maxImages= */ 1);
frameProcessorChain = frameProcessorChain =
FrameProcessorChain.create( new FrameProcessorChain(
context, context,
PIXEL_WIDTH_HEIGHT_RATIO, PIXEL_WIDTH_HEIGHT_RATIO,
frameProcessorsList, frameProcessorsList,
sizes, sizes,
outputImageReader.getSurface(), /* enableExperimentalHdrEditing= */ false);
/* enableExperimentalHdrEditing= */ false, frameProcessorChain.configure(
Transformer.DebugViewProvider.NONE); outputImageReader.getSurface(), outputWidth, outputHeight, /* debugSurfaceView= */ null);
frameProcessorChain.registerInputFrame(); frameProcessorChain.registerInputFrame();
// Queue the first video frame from the extractor. // Queue the first video frame from the extractor.
String mimeType = checkNotNull(mediaFormat.getString(MediaFormat.KEY_MIME)); String mimeType = checkNotNull(mediaFormat.getString(MediaFormat.KEY_MIME));
mediaCodec = MediaCodec.createDecoderByType(mimeType); mediaCodec = MediaCodec.createDecoderByType(mimeType);
mediaCodec.configure( mediaCodec.configure(
mediaFormat, mediaFormat, frameProcessorChain.getInputSurface(), /* crypto= */ null, /* flags= */ 0);
frameProcessorChain.createInputSurface(),
/* crypto= */ null,
/* flags= */ 0);
mediaCodec.start(); mediaCodec.start();
int inputBufferIndex = mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US); int inputBufferIndex = mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US);
assertThat(inputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER); assertThat(inputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER);
......
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameProcessorChainTest.java deleted 100644 → 0
/*
* Copyright 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.transformer;
import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;
import android.content.Context;
import android.graphics.SurfaceTexture;
import android.util.Size;
import android.view.Surface;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList;
import java.util.List;
import org.junit.Test;
import org.junit.runner.RunWith;
/**
* Test for {@link FrameProcessorChain#create(Context, float, List, List, Surface, boolean,
* Transformer.DebugViewProvider) creating} a {@link FrameProcessorChain}.
*/
@RunWith(AndroidJUnit4.class)
public final class FrameProcessorChainTest {
// TODO(b/212539951): Make this a robolectric test by e.g. updating shadows or adding a
// wrapper around GlUtil to allow the usage of mocks or fakes which don't need (Shadow)GLES20.
@Test
public void create_withSupportedPixelWidthHeightRatio_completesSuccessfully()
throws TransformationException {
Context context = getApplicationContext();
FrameProcessorChain.create(
context,
/* pixelWidthHeightRatio= */ 1,
/* frameProcessors= */ ImmutableList.of(),
/* sizes= */ ImmutableList.of(new Size(200, 100)),
/* outputSurface= */ new Surface(new SurfaceTexture(false)),
/* enableExperimentalHdrEditing= */ false,
Transformer.DebugViewProvider.NONE);
}
@Test
public void create_withUnsupportedPixelWidthHeightRatio_throwsException() {
Context context = getApplicationContext();
TransformationException exception =
assertThrows(
TransformationException.class,
() ->
FrameProcessorChain.create(
context,
/* pixelWidthHeightRatio= */ 2,
/* frameProcessors= */ ImmutableList.of(),
/* sizes= */ ImmutableList.of(new Size(200, 100)),
/* outputSurface= */ new Surface(new SurfaceTexture(false)),
/* enableExperimentalHdrEditing= */ false,
Transformer.DebugViewProvider.NONE));
assertThat(exception).hasCauseThat().isInstanceOf(UnsupportedOperationException.class);
assertThat(exception).hasCauseThat().hasMessageThat().contains("pixelWidthHeightRatio");
}
}
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameProcessorChain.java
...@@ -18,6 +18,7 @@ package com.google.android.exoplayer2.transformer; ...@@ -18,6 +18,7 @@ package com.google.android.exoplayer2.transformer;
import static com.google.android.exoplayer2.util.Assertions.checkArgument; import static com.google.android.exoplayer2.util.Assertions.checkArgument;
import static com.google.android.exoplayer2.util.Assertions.checkNotNull; import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import static com.google.android.exoplayer2.util.Assertions.checkState; import static com.google.android.exoplayer2.util.Assertions.checkState;
import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
import static com.google.common.collect.Iterables.getLast; import static com.google.common.collect.Iterables.getLast;
import android.content.Context; import android.content.Context;
...@@ -44,18 +45,19 @@ import java.util.concurrent.ExecutorService; ...@@ -44,18 +45,19 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import org.checkerframework.checker.nullness.qual.EnsuresNonNull;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.checker.nullness.qual.RequiresNonNull; import org.checkerframework.checker.nullness.qual.RequiresNonNull;
/** /**
* {@code FrameProcessorChain} applies changes to individual video frames. * {@code FrameProcessorChain} applies changes to individual video frames.
* *
* <p>Input becomes available on its {@link #createInputSurface() input surface} asynchronously and * <p>Input becomes available on its {@link #getInputSurface() input surface} asynchronously and is
* is processed on a background thread as it becomes available. All input frames should be {@link * processed on a background thread as it becomes available. All input frames should be {@link
* #registerInputFrame() registered} before they are rendered to the input surface. {@link * #registerInputFrame() registered} before they are rendered to the input surface. {@link
* #hasPendingFrames()} can be used to check whether there are frames that have not been fully * #hasPendingFrames()} can be used to check whether there are frames that have not been fully
* processed yet. The {@code FrameProcessorChain} writes output to the surface passed to {@link * processed yet. The {@code FrameProcessorChain} writes output to the surface passed to {@link
* #create(Context, float, List, List, Surface, boolean, Transformer.DebugViewProvider)}. * #configure(Surface, int, int, SurfaceView)}.
*/ */
/* package */ final class FrameProcessorChain { /* package */ final class FrameProcessorChain {
...@@ -89,170 +91,15 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -89,170 +91,15 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
return sizes; return sizes;
} }
/**
* Returns a new {@code FrameProcessorChain} for applying changes to individual frames.
*
* @param context A {@link Context}.
* @param pixelWidthHeightRatio The ratio of width over height, for each pixel.
* @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors} to apply to each frame.
* Their output sizes must be {@link GlFrameProcessor#configureOutputSize(int, int)}
* configured}.
* @param sizes The input {@link Size} as well as the output {@link Size} of each {@link
* GlFrameProcessor}.
* @param outputSurface The {@link Surface}.
* @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal.
* @param debugViewProvider Provider for optional debug views to show intermediate output.
* @return A configured {@code FrameProcessorChain}.
* @throws TransformationException If the {@code pixelWidthHeightRatio} isn't 1, reading shader
* files fails, or an OpenGL error occurs while creating and configuring the OpenGL
* components.
*/
public static FrameProcessorChain create(
Context context,
float pixelWidthHeightRatio,
List<GlFrameProcessor> frameProcessors,
List<Size> sizes,
Surface outputSurface,
boolean enableExperimentalHdrEditing,
Transformer.DebugViewProvider debugViewProvider)
throws TransformationException {
checkArgument(frameProcessors.size() + 1 == sizes.size());
if (pixelWidthHeightRatio != 1.0f) {
// TODO(b/211782176): Consider implementing support for non-square pixels.
throw TransformationException.createForFrameProcessorChain(
new UnsupportedOperationException(
"Transformer's FrameProcessorChain currently does not support frame edits on"
+ " non-square pixels. The pixelWidthHeightRatio is: "
+ pixelWidthHeightRatio),
TransformationException.ERROR_CODE_GL_INIT_FAILED);
}
@Nullable
SurfaceView debugSurfaceView =
debugViewProvider.getDebugPreviewSurfaceView(
getLast(sizes).getWidth(), getLast(sizes).getHeight());
int debugPreviewWidth;
int debugPreviewHeight;
if (debugSurfaceView != null) {
debugPreviewWidth = debugSurfaceView.getWidth();
debugPreviewHeight = debugSurfaceView.getHeight();
} else {
debugPreviewWidth = C.LENGTH_UNSET;
debugPreviewHeight = C.LENGTH_UNSET;
}
ExternalCopyFrameProcessor externalCopyFrameProcessor =
new ExternalCopyFrameProcessor(context, enableExperimentalHdrEditing);
ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME);
Future<FrameProcessorChain> frameProcessorChainFuture =
singleThreadExecutorService.submit(
() ->
createOpenGlObjectsAndFrameProcessorChain(
singleThreadExecutorService,
externalCopyFrameProcessor,
frameProcessors,
sizes,
outputSurface,
enableExperimentalHdrEditing,
debugSurfaceView,
debugPreviewWidth,
debugPreviewHeight));
try {
return frameProcessorChainFuture.get();
} catch (ExecutionException e) {
throw TransformationException.createForFrameProcessorChain(
e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw TransformationException.createForFrameProcessorChain(
e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
}
}
/**
* Creates a {@code FrameProcessorChain} and its OpenGL objects.
*
* <p>As the {@code FrameProcessorChain} will call OpenGL commands on the {@code
* singleThreadExecutorService}'s thread, the OpenGL context and objects also need to be created
* on that thread. So this method should only be called on the {@code
* singleThreadExecutorService}'s thread.
*/
private static FrameProcessorChain createOpenGlObjectsAndFrameProcessorChain(
ExecutorService singleThreadExecutorService,
ExternalCopyFrameProcessor externalCopyFrameProcessor,
List<GlFrameProcessor> frameProcessors,
List<Size> sizes,
Surface outputSurface,
boolean enableExperimentalHdrEditing,
@Nullable SurfaceView debugSurfaceView,
int debugPreviewWidth,
int debugPreviewHeight)
throws IOException {
EGLDisplay eglDisplay = GlUtil.createEglDisplay();
final EGLContext eglContext;
final EGLSurface eglSurface;
@Nullable EGLSurface debugPreviewEglSurface = null;
if (enableExperimentalHdrEditing) {
eglContext = GlUtil.createEglContextEs3Rgba1010102(eglDisplay);
// TODO(b/209404935): Don't assume BT.2020 PQ input/output.
eglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, outputSurface);
if (debugSurfaceView != null) {
debugPreviewEglSurface =
GlUtil.getEglSurfaceBt2020Pq(eglDisplay, checkNotNull(debugSurfaceView.getHolder()));
}
} else {
eglContext = GlUtil.createEglContext(eglDisplay);
eglSurface = GlUtil.getEglSurface(eglDisplay, outputSurface);
if (debugSurfaceView != null) {
debugPreviewEglSurface =
GlUtil.getEglSurface(eglDisplay, checkNotNull(debugSurfaceView.getHolder()));
}
}
GlUtil.focusEglSurface(
eglDisplay, eglContext, eglSurface, getLast(sizes).getWidth(), getLast(sizes).getHeight());
int inputExternalTexId = GlUtil.createExternalTexture();
externalCopyFrameProcessor.configureOutputSize(
/* inputWidth= */ sizes.get(0).getWidth(), /* inputHeight= */ sizes.get(0).getHeight());
externalCopyFrameProcessor.initialize(inputExternalTexId);
int[] framebuffers = new int[frameProcessors.size()];
for (int i = 0; i < frameProcessors.size(); i++) {
int inputTexId = GlUtil.createTexture(sizes.get(i).getWidth(), sizes.get(i).getHeight());
framebuffers[i] = GlUtil.createFboForTexture(inputTexId);
frameProcessors.get(i).initialize(inputTexId);
}
return new FrameProcessorChain(
singleThreadExecutorService,
eglDisplay,
eglContext,
eglSurface,
externalCopyFrameProcessor,
frameProcessors,
inputExternalTexId,
framebuffers,
sizes,
debugPreviewEglSurface,
debugPreviewWidth,
debugPreviewHeight);
}
private static final String THREAD_NAME = "Transformer:FrameProcessorChain"; private static final String THREAD_NAME = "Transformer:FrameProcessorChain";
private final EGLContext eglContext; private final boolean enableExperimentalHdrEditing;
private final EGLDisplay eglDisplay; private final EGLDisplay eglDisplay;
/** private final EGLContext eglContext;
* Wraps the output {@link Surface} that is populated with the output of the final {@link
* GlFrameProcessor} for each frame.
*/
private final EGLSurface eglSurface;
/** Some OpenGL commands may block, so all OpenGL commands are run on a background thread. */ /** Some OpenGL commands may block, so all OpenGL commands are run on a background thread. */
private final ExecutorService singleThreadExecutorService; private final ExecutorService singleThreadExecutorService;
/** The {@link #singleThreadExecutorService} thread. */
private @MonotonicNonNull Thread glThread;
/** Futures corresponding to the executor service's pending tasks. */ /** Futures corresponding to the executor service's pending tasks. */
private final ConcurrentLinkedQueue<Future<?>> futures; private final ConcurrentLinkedQueue<Future<?>> futures;
/** Number of frames {@link #registerInputFrame() registered} but not fully processed. */ /** Number of frames {@link #registerInputFrame() registered} but not fully processed. */
...@@ -268,8 +115,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -268,8 +115,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
/** /**
* Identifier of the external texture the {@link ExternalCopyFrameProcessor} reads its input from. * Identifier of the external texture the {@link ExternalCopyFrameProcessor} reads its input from.
*/ */
private final int inputExternalTexId; private int inputExternalTexId;
/** Transformation matrix associated with the surface texture. */ /** Transformation matrix associated with the {@link #inputSurfaceTexture}. */
private final float[] textureTransformMatrix; private final float[] textureTransformMatrix;
private final ExternalCopyFrameProcessor externalCopyFrameProcessor; private final ExternalCopyFrameProcessor externalCopyFrameProcessor;
...@@ -289,55 +136,129 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -289,55 +136,129 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
*/ */
private final List<Size> sizes; private final List<Size> sizes;
private final int debugPreviewWidth; private int outputWidth;
private final int debugPreviewHeight; private int outputHeight;
/**
* Wraps the output {@link Surface} that is populated with the output of the final {@link
* GlFrameProcessor} for each frame.
*/
private @MonotonicNonNull EGLSurface eglSurface;
private int debugPreviewWidth;
private int debugPreviewHeight;
/** /**
* Wraps a debug {@link SurfaceView} that is populated with the output of the final {@link * Wraps a debug {@link SurfaceView} that is populated with the output of the final {@link
* GlFrameProcessor} for each frame. * GlFrameProcessor} for each frame.
*/ */
@Nullable private final EGLSurface debugPreviewEglSurface; private @MonotonicNonNull EGLSurface debugPreviewEglSurface;
private FrameProcessorChain( /**
ExecutorService singleThreadExecutorService, * Creates a new instance.
EGLDisplay eglDisplay, *
EGLContext eglContext, * @param context A {@link Context}.
EGLSurface eglSurface, * @param pixelWidthHeightRatio The ratio of width over height, for each pixel.
ExternalCopyFrameProcessor externalCopyFrameProcessor, * @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors} to apply to each frame.
* Their output sizes must be {@link GlFrameProcessor#configureOutputSize(int, int)}
* configured}.
* @param sizes The input {@link Size} as well as the output {@link Size} of each {@link
* GlFrameProcessor}.
* @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal.
* @throws TransformationException If the {@code pixelWidthHeightRatio} isn't 1.
*/
public FrameProcessorChain(
Context context,
float pixelWidthHeightRatio,
List<GlFrameProcessor> frameProcessors, List<GlFrameProcessor> frameProcessors,
int inputExternalTexId,
int[] framebuffers,
List<Size> sizes, List<Size> sizes,
@Nullable EGLSurface debugPreviewEglSurface, boolean enableExperimentalHdrEditing)
int debugPreviewWidth, throws TransformationException {
int debugPreviewHeight) { checkArgument(frameProcessors.size() + 1 == sizes.size());
this.singleThreadExecutorService = singleThreadExecutorService;
this.eglDisplay = eglDisplay; if (pixelWidthHeightRatio != 1.0f) {
this.eglContext = eglContext; // TODO(b/211782176): Consider implementing support for non-square pixels.
this.eglSurface = eglSurface; throw TransformationException.createForFrameProcessorChain(
this.externalCopyFrameProcessor = externalCopyFrameProcessor; new UnsupportedOperationException(
"Transformer's FrameProcessorChain currently does not support frame edits on"
+ " non-square pixels. The pixelWidthHeightRatio is: "
+ pixelWidthHeightRatio),
TransformationException.ERROR_CODE_GL_INIT_FAILED);
}
this.enableExperimentalHdrEditing = enableExperimentalHdrEditing;
this.frameProcessors = frameProcessors; this.frameProcessors = frameProcessors;
this.inputExternalTexId = inputExternalTexId;
this.framebuffers = framebuffers;
this.sizes = sizes; this.sizes = sizes;
this.debugPreviewEglSurface = debugPreviewEglSurface;
this.debugPreviewWidth = debugPreviewWidth;
this.debugPreviewHeight = debugPreviewHeight;
try {
eglDisplay = GlUtil.createEglDisplay();
eglContext =
enableExperimentalHdrEditing
? GlUtil.createEglContextEs3Rgba1010102(eglDisplay)
: GlUtil.createEglContext(eglDisplay);
} catch (GlUtil.GlException e) {
throw TransformationException.createForFrameProcessorChain(
e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
}
singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME);
futures = new ConcurrentLinkedQueue<>(); futures = new ConcurrentLinkedQueue<>();
pendingFrameCount = new AtomicInteger(); pendingFrameCount = new AtomicInteger();
textureTransformMatrix = new float[16]; textureTransformMatrix = new float[16];
externalCopyFrameProcessor =
new ExternalCopyFrameProcessor(context, enableExperimentalHdrEditing);
framebuffers = new int[frameProcessors.size()];
outputWidth = getLast(sizes).getWidth();
outputHeight = getLast(sizes).getHeight();
debugPreviewWidth = C.LENGTH_UNSET;
debugPreviewHeight = C.LENGTH_UNSET;
} }
/** /**
* Creates the input {@link Surface} and configures it to process frames. * Configures the {@code FrameProcessorChain} to process frames to the specified output targets.
* *
* <p>This method must not be called again after creating an input surface. * <p>This method may only be called once and may override the {@link
* GlFrameProcessor#configureOutputSize(int, int) output size} of the final {@link
* GlFrameProcessor}.
* *
* @return The configured input {@link Surface}. * @param outputSurface The output {@link Surface}.
* @throws IllegalStateException If an input {@link Surface} has already been created. * @param outputWidth The output width, in pixels.
* @param outputHeight The output height, in pixels.
* @param debugSurfaceView Optional debug {@link SurfaceView} to show output.
* @throws IllegalStateException If the {@code FrameProcessorChain} has already been configured.
* @throws TransformationException If reading shader files fails, or an OpenGL error occurs while
* creating and configuring the OpenGL components.
*/ */
public Surface createInputSurface() { public void configure(
checkState(inputSurface == null, "The input surface has already been created."); Surface outputSurface,
int outputWidth,
int outputHeight,
@Nullable SurfaceView debugSurfaceView)
throws TransformationException {
checkState(inputSurface == null, "The FrameProcessorChain has already been configured.");
// TODO(b/218488308): Don't override output size for encoder fallback. Instead allow the final
// GlFrameProcessor to be re-configured or append another GlFrameProcessor.
this.outputWidth = outputWidth;
this.outputHeight = outputHeight;
if (debugSurfaceView != null) {
debugPreviewWidth = debugSurfaceView.getWidth();
debugPreviewHeight = debugSurfaceView.getHeight();
}
try {
// Wait for task to finish to be able to use inputExternalTexId to create the SurfaceTexture.
singleThreadExecutorService
.submit(
() ->
createOpenGlObjectsAndInitializeFrameProcessors(outputSurface, debugSurfaceView))
.get();
} catch (ExecutionException e) {
throw TransformationException.createForFrameProcessorChain(
e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw TransformationException.createForFrameProcessorChain(
e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
}
inputSurfaceTexture = new SurfaceTexture(inputExternalTexId); inputSurfaceTexture = new SurfaceTexture(inputExternalTexId);
inputSurfaceTexture.setOnFrameAvailableListener( inputSurfaceTexture.setOnFrameAvailableListener(
surfaceTexture -> { surfaceTexture -> {
...@@ -355,6 +276,16 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -355,6 +276,16 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
} }
}); });
inputSurface = new Surface(inputSurfaceTexture); inputSurface = new Surface(inputSurfaceTexture);
}
/**
* Returns the input {@link Surface}.
*
* <p>The {@code FrameProcessorChain} must be {@link #configure(Surface, int, int, SurfaceView)
* configured}.
*/
public Surface getInputSurface() {
checkStateNotNull(inputSurface, "The FrameProcessorChain must be configured.");
return inputSurface; return inputSurface;
} }
...@@ -440,16 +371,60 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -440,16 +371,60 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
singleThreadExecutorService.shutdown(); singleThreadExecutorService.shutdown();
} }
/** Processes an input frame. */ /**
@RequiresNonNull("inputSurfaceTexture") * Creates the OpenGL textures, framebuffers, surfaces, and initializes the {@link
* GlFrameProcessor GlFrameProcessors}.
*
* <p>This method must by executed on the same thread as {@link #processFrame()}, i.e., executed
* by the {@link #singleThreadExecutorService}.
*/
@EnsuresNonNull("eglSurface")
private Void createOpenGlObjectsAndInitializeFrameProcessors(
Surface outputSurface, @Nullable SurfaceView debugSurfaceView) throws IOException {
glThread = Thread.currentThread();
if (enableExperimentalHdrEditing) {
// TODO(b/209404935): Don't assume BT.2020 PQ input/output.
eglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, outputSurface);
if (debugSurfaceView != null) {
debugPreviewEglSurface =
GlUtil.getEglSurfaceBt2020Pq(eglDisplay, checkNotNull(debugSurfaceView.getHolder()));
}
} else {
eglSurface = GlUtil.getEglSurface(eglDisplay, outputSurface);
if (debugSurfaceView != null) {
debugPreviewEglSurface =
GlUtil.getEglSurface(eglDisplay, checkNotNull(debugSurfaceView.getHolder()));
}
}
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight);
inputExternalTexId = GlUtil.createExternalTexture();
externalCopyFrameProcessor.configureOutputSize(
/* inputWidth= */ sizes.get(0).getWidth(), /* inputHeight= */ sizes.get(0).getHeight());
externalCopyFrameProcessor.initialize(inputExternalTexId);
for (int i = 0; i < frameProcessors.size(); i++) {
int inputTexId = GlUtil.createTexture(sizes.get(i).getWidth(), sizes.get(i).getHeight());
framebuffers[i] = GlUtil.createFboForTexture(inputTexId);
frameProcessors.get(i).initialize(inputTexId);
}
// Return something because only Callables not Runnables can throw checked exceptions.
return null;
}
/**
* Processes an input frame.
*
* <p>This method must by executed on the same thread as {@link
* #createOpenGlObjectsAndInitializeFrameProcessors(Surface,SurfaceView)}, i.e., executed by the
* {@link #singleThreadExecutorService}.
*/
@RequiresNonNull({"inputSurfaceTexture", "eglSurface"})
private void processFrame() { private void processFrame() {
inputSurfaceTexture.updateTexImage(); checkState(Thread.currentThread().equals(glThread));
inputSurfaceTexture.getTransformMatrix(textureTransformMatrix);
long presentationTimeNs = inputSurfaceTexture.getTimestamp();
if (frameProcessors.isEmpty()) { if (frameProcessors.isEmpty()) {
GlUtil.focusEglSurface( GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight);
eglDisplay, eglContext, eglSurface, sizes.get(0).getWidth(), sizes.get(0).getHeight());
} else { } else {
GlUtil.focusFramebuffer( GlUtil.focusFramebuffer(
eglDisplay, eglDisplay,
...@@ -459,7 +434,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -459,7 +434,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
sizes.get(0).getWidth(), sizes.get(0).getWidth(),
sizes.get(0).getHeight()); sizes.get(0).getHeight());
} }
inputSurfaceTexture.updateTexImage();
inputSurfaceTexture.getTransformMatrix(textureTransformMatrix);
externalCopyFrameProcessor.setTextureTransformMatrix(textureTransformMatrix); externalCopyFrameProcessor.setTextureTransformMatrix(textureTransformMatrix);
long presentationTimeNs = inputSurfaceTexture.getTimestamp();
externalCopyFrameProcessor.updateProgramAndDraw(presentationTimeNs); externalCopyFrameProcessor.updateProgramAndDraw(presentationTimeNs);
for (int i = 0; i < frameProcessors.size() - 1; i++) { for (int i = 0; i < frameProcessors.size() - 1; i++) {
...@@ -473,12 +451,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -473,12 +451,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
frameProcessors.get(i).updateProgramAndDraw(presentationTimeNs); frameProcessors.get(i).updateProgramAndDraw(presentationTimeNs);
} }
if (!frameProcessors.isEmpty()) { if (!frameProcessors.isEmpty()) {
GlUtil.focusEglSurface( GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight);
eglDisplay,
eglContext,
eglSurface,
getLast(sizes).getWidth(),
getLast(sizes).getHeight());
getLast(frameProcessors).updateProgramAndDraw(presentationTimeNs); getLast(frameProcessors).updateProgramAndDraw(presentationTimeNs);
} }
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java
...@@ -110,24 +110,22 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -110,24 +110,22 @@ import org.checkerframework.dataflow.qual.Pure;
requestedEncoderFormat, requestedEncoderFormat,
encoderSupportedFormat)); encoderSupportedFormat));
// TODO(b/218488308): Allow the final GlFrameProcessor to be re-configured if its output size
// has to change due to encoder fallback or append another GlFrameProcessor.
frameProcessorSizes.set(
frameProcessorSizes.size() - 1,
new Size(encoderSupportedFormat.width, encoderSupportedFormat.height));
frameProcessorChain = frameProcessorChain =
FrameProcessorChain.create( new FrameProcessorChain(
context, context,
inputFormat.pixelWidthHeightRatio, inputFormat.pixelWidthHeightRatio,
frameProcessors, frameProcessors,
frameProcessorSizes, frameProcessorSizes,
transformationRequest.enableHdrEditing);
frameProcessorChain.configure(
/* outputSurface= */ encoder.getInputSurface(), /* outputSurface= */ encoder.getInputSurface(),
transformationRequest.enableHdrEditing, /* outputWidth= */ encoderSupportedFormat.width,
debugViewProvider); /* outputHeight= */ encoderSupportedFormat.height,
debugViewProvider.getDebugPreviewSurfaceView(
encoderSupportedFormat.width, encoderSupportedFormat.height));
decoder = decoder =
decoderFactory.createForVideoDecoding( decoderFactory.createForVideoDecoding(inputFormat, frameProcessorChain.getInputSurface());
inputFormat, frameProcessorChain.createInputSurface());
} }
@Override @Override
......
library/transformer/src/test/java/com/google/android/exoplayer2/transformer/FrameProcessorChainTest.java
...@@ -15,8 +15,11 @@ ...@@ -15,8 +15,11 @@
*/ */
package com.google.android.exoplayer2.transformer; package com.google.android.exoplayer2.transformer;
import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;
import android.content.Context;
import android.util.Size; import android.util.Size;
import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
...@@ -27,11 +30,42 @@ import org.junit.runner.RunWith; ...@@ -27,11 +30,42 @@ import org.junit.runner.RunWith;
/** /**
* Robolectric tests for {@link FrameProcessorChain}. * Robolectric tests for {@link FrameProcessorChain}.
* *
* <p>See {@code FrameProcessorChainTest} and {@code FrameProcessorChainPixelTest} in the * <p>See {@code FrameProcessorChainPixelTest} in the androidTest directory for instrumentation
* androidTest directory for instrumentation tests. * tests.
*/ */
@RunWith(AndroidJUnit4.class) @RunWith(AndroidJUnit4.class)
public final class FrameProcessorChainTest { public final class FrameProcessorChainTest {
@Test
public void construct_withSupportedPixelWidthHeightRatio_completesSuccessfully()
throws TransformationException {
Context context = getApplicationContext();
new FrameProcessorChain(
context,
/* pixelWidthHeightRatio= */ 1,
/* frameProcessors= */ ImmutableList.of(),
/* sizes= */ ImmutableList.of(new Size(200, 100)),
/* enableExperimentalHdrEditing= */ false);
}
@Test
public void construct_withUnsupportedPixelWidthHeightRatio_throwsException() {
Context context = getApplicationContext();
TransformationException exception =
assertThrows(
TransformationException.class,
() ->
new FrameProcessorChain(
context,
/* pixelWidthHeightRatio= */ 2,
/* frameProcessors= */ ImmutableList.of(),
/* sizes= */ ImmutableList.of(new Size(200, 100)),
/* enableExperimentalHdrEditing= */ false));
assertThat(exception).hasCauseThat().isInstanceOf(UnsupportedOperationException.class);
assertThat(exception).hasCauseThat().hasMessageThat().contains("pixelWidthHeightRatio");
}
@Test @Test
public void configureOutputDimensions_withEmptyList_returnsInputSize() { public void configureOutputDimensions_withEmptyList_returnsInputSize() {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment