Skip to content

SDK / exoplayer

Go to a project
Commit f08fdf6a by hschlueter Committed by Ian Baker
Options

Add listener for FrameProcessingExceptions. 

This listener replaces
FrameProcessorChain#getAndRethrowBackgroundExceptions.
The listener uses a new exception type FrameProcessingException
separate from TransformationException as the frame processing
components will be made reusable outside of transformer soon.

PiperOrigin-RevId: 447455746
parent e89189e3
Expand all
Inline Side-by-side
Showing with 282 additions and 59 deletions
demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/BitmapOverlayFrameProcessor.java
...@@ -29,6 +29,7 @@ import android.opengl.GLES20; ...@@ -29,6 +29,7 @@ import android.opengl.GLES20;
import android.opengl.GLUtils; import android.opengl.GLUtils;
import android.util.Size; import android.util.Size;
import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.transformer.FrameProcessingException;
import com.google.android.exoplayer2.transformer.GlFrameProcessor; import com.google.android.exoplayer2.transformer.GlFrameProcessor;
import com.google.android.exoplayer2.util.GlProgram; import com.google.android.exoplayer2.util.GlProgram;
import com.google.android.exoplayer2.util.GlUtil; import com.google.android.exoplayer2.util.GlUtil;
...@@ -116,9 +117,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -116,9 +117,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
@Override @Override
public void drawFrame(long presentationTimeUs) { public void drawFrame(long presentationTimeUs) throws FrameProcessingException {
checkStateNotNull(glProgram); try {
glProgram.use(); checkStateNotNull(glProgram).use();
// Draw to the canvas and store it in a texture. // Draw to the canvas and store it in a texture.
String text = String text =
...@@ -138,6 +139,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -138,6 +139,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
glProgram.bindAttributesAndUniforms(); glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad. // The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
GlUtil.checkGlError();
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e);
}
} }
@Override @Override
......
demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/PeriodicVignetteFrameProcessor.java
...@@ -21,6 +21,7 @@ import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; ...@@ -21,6 +21,7 @@ import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
import android.content.Context; import android.content.Context;
import android.opengl.GLES20; import android.opengl.GLES20;
import android.util.Size; import android.util.Size;
import com.google.android.exoplayer2.transformer.FrameProcessingException;
import com.google.android.exoplayer2.transformer.GlFrameProcessor; import com.google.android.exoplayer2.transformer.GlFrameProcessor;
import com.google.android.exoplayer2.util.GlProgram; import com.google.android.exoplayer2.util.GlProgram;
import com.google.android.exoplayer2.util.GlUtil; import com.google.android.exoplayer2.util.GlUtil;
...@@ -98,14 +99,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -98,14 +99,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
@Override @Override
public void drawFrame(long presentationTimeUs) { public void drawFrame(long presentationTimeUs) throws FrameProcessingException {
try {
checkStateNotNull(glProgram).use(); checkStateNotNull(glProgram).use();
double theta = presentationTimeUs * 2 * Math.PI / DIMMING_PERIOD_US; double theta = presentationTimeUs * 2 * Math.PI / DIMMING_PERIOD_US;
float innerRadius = minInnerRadius + deltaInnerRadius * (0.5f - 0.5f * (float) Math.cos(theta)); float innerRadius =
minInnerRadius + deltaInnerRadius * (0.5f - 0.5f * (float) Math.cos(theta));
glProgram.setFloatsUniform("uInnerRadius", new float[] {innerRadius}); glProgram.setFloatsUniform("uInnerRadius", new float[] {innerRadius});
glProgram.bindAttributesAndUniforms(); glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad. // The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e);
}
} }
@Override @Override
......
demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeFrameProcessor.java
...@@ -22,6 +22,7 @@ import android.content.Context; ...@@ -22,6 +22,7 @@ import android.content.Context;
import android.opengl.EGL14; import android.opengl.EGL14;
import android.opengl.GLES20; import android.opengl.GLES20;
import android.util.Size; import android.util.Size;
import com.google.android.exoplayer2.transformer.FrameProcessingException;
import com.google.android.exoplayer2.transformer.GlFrameProcessor; import com.google.android.exoplayer2.transformer.GlFrameProcessor;
import com.google.android.exoplayer2.util.ConditionVariable; import com.google.android.exoplayer2.util.ConditionVariable;
import com.google.android.exoplayer2.util.GlProgram; import com.google.android.exoplayer2.util.GlProgram;
...@@ -112,7 +113,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -112,7 +113,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
@Override @Override
public void drawFrame(long presentationTimeUs) { public void drawFrame(long presentationTimeUs) throws FrameProcessingException {
frameProcessorConditionVariable.close(); frameProcessorConditionVariable.close();
// Pass the input frame to MediaPipe. // Pass the input frame to MediaPipe.
...@@ -133,7 +134,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -133,7 +134,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
if (frameProcessorPendingError != null) { if (frameProcessorPendingError != null) {
throw new IllegalStateException(frameProcessorPendingError); throw new FrameProcessingException(frameProcessorPendingError);
} }
// Copy from MediaPipe's output texture to the current output. // Copy from MediaPipe's output texture to the current output.
...@@ -148,6 +149,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -148,6 +149,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
glProgram.bindAttributesAndUniforms(); glProgram.bindAttributesAndUniforms();
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
GlUtil.checkGlError(); GlUtil.checkGlError();
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e);
} finally { } finally {
checkStateNotNull(outputFrame).release(); checkStateNotNull(outputFrame).release();
} }
......
library/common/src/main/java/com/google/android/exoplayer2/util/GlUtil.java
...@@ -49,6 +49,8 @@ public final class GlUtil { ...@@ -49,6 +49,8 @@ public final class GlUtil {
} }
} }
// TODO(b/231937416): Consider removing this flag, enabling assertions by default, and making
// GlException checked.
/** Whether to throw a {@link GlException} in case of an OpenGL error. */ /** Whether to throw a {@link GlException} in case of an OpenGL error. */
public static boolean glAssertionsEnabled = false; public static boolean glAssertionsEnabled = false;
......
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameProcessorChainPixelTest.java
...@@ -18,6 +18,7 @@ package com.google.android.exoplayer2.transformer; ...@@ -18,6 +18,7 @@ package com.google.android.exoplayer2.transformer;
import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
import static com.google.android.exoplayer2.transformer.BitmapTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE; import static com.google.android.exoplayer2.transformer.BitmapTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE;
import static com.google.android.exoplayer2.util.Assertions.checkNotNull; import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertThat;
import static java.util.Arrays.asList; import static java.util.Arrays.asList;
...@@ -36,6 +37,7 @@ import androidx.annotation.Nullable; ...@@ -36,6 +37,7 @@ import androidx.annotation.Nullable;
import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.MimeTypes;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.concurrent.atomic.AtomicReference;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.junit.After; import org.junit.After;
import org.junit.Test; import org.junit.Test;
...@@ -78,6 +80,9 @@ public final class FrameProcessorChainPixelTest { ...@@ -78,6 +80,9 @@ public final class FrameProcessorChainPixelTest {
/** The ratio of width over height, for each pixel in a frame. */ /** The ratio of width over height, for each pixel in a frame. */
private static final float DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO = 1; private static final float DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO = 1;
private final AtomicReference<FrameProcessingException> frameProcessingException =
new AtomicReference<>();
private @MonotonicNonNull FrameProcessorChain frameProcessorChain; private @MonotonicNonNull FrameProcessorChain frameProcessorChain;
private @MonotonicNonNull ImageReader outputImageReader; private @MonotonicNonNull ImageReader outputImageReader;
private @MonotonicNonNull MediaFormat mediaFormat; private @MonotonicNonNull MediaFormat mediaFormat;
...@@ -229,6 +234,15 @@ public final class FrameProcessorChainPixelTest { ...@@ -229,6 +234,15 @@ public final class FrameProcessorChainPixelTest {
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
} }
@Test
public void processData_withFrameProcessingException_callsListener() throws Exception {
setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, ThrowingFrameProcessor::new);
Thread.sleep(FRAME_PROCESSING_WAIT_MS);
assertThat(frameProcessingException.get()).isNotNull();
}
/** /**
* Set up and prepare the first frame from an input video, as well as relevant test * Set up and prepare the first frame from an input video, as well as relevant test
* infrastructure. The frame will be sent towards the {@link FrameProcessorChain}, and may be * infrastructure. The frame will be sent towards the {@link FrameProcessorChain}, and may be
...@@ -258,6 +272,7 @@ public final class FrameProcessorChainPixelTest { ...@@ -258,6 +272,7 @@ public final class FrameProcessorChainPixelTest {
frameProcessorChain = frameProcessorChain =
FrameProcessorChain.create( FrameProcessorChain.create(
context, context,
/* listener= */ this.frameProcessingException::set,
pixelWidthHeightRatio, pixelWidthHeightRatio,
inputWidth, inputWidth,
inputHeight, inputHeight,
...@@ -321,11 +336,11 @@ public final class FrameProcessorChainPixelTest { ...@@ -321,11 +336,11 @@ public final class FrameProcessorChainPixelTest {
} }
} }
private Bitmap processFirstFrameAndEnd() throws InterruptedException, TransformationException { private Bitmap processFirstFrameAndEnd() throws InterruptedException {
checkNotNull(frameProcessorChain).signalEndOfInputStream(); checkNotNull(frameProcessorChain).signalEndOfInputStream();
Thread.sleep(FRAME_PROCESSING_WAIT_MS); Thread.sleep(FRAME_PROCESSING_WAIT_MS);
assertThat(frameProcessorChain.isEnded()).isTrue(); assertThat(frameProcessorChain.isEnded()).isTrue();
frameProcessorChain.getAndRethrowBackgroundExceptions(); assertThat(frameProcessingException.get()).isNull();
Image frameProcessorChainOutputImage = checkNotNull(outputImageReader).acquireLatestImage(); Image frameProcessorChainOutputImage = checkNotNull(outputImageReader).acquireLatestImage();
Bitmap actualBitmap = Bitmap actualBitmap =
...@@ -333,4 +348,27 @@ public final class FrameProcessorChainPixelTest { ...@@ -333,4 +348,27 @@ public final class FrameProcessorChainPixelTest {
frameProcessorChainOutputImage.close(); frameProcessorChainOutputImage.close();
return actualBitmap; return actualBitmap;
} }
private static class ThrowingFrameProcessor implements GlFrameProcessor {
private @MonotonicNonNull Size outputSize;
@Override
public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight) {
outputSize = new Size(inputWidth, inputHeight);
}
@Override
public Size getOutputSize() {
return checkStateNotNull(outputSize);
}
@Override
public void drawFrame(long presentationTimeUs) throws FrameProcessingException {
throw new FrameProcessingException("An exception occurred.");
}
@Override
public void release() {}
}
} }
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameProcessorChainTest.java
...@@ -23,6 +23,7 @@ import android.util.Size; ...@@ -23,6 +23,7 @@ import android.util.Size;
import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import java.util.List; import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
...@@ -33,6 +34,8 @@ import org.junit.runner.RunWith; ...@@ -33,6 +34,8 @@ import org.junit.runner.RunWith;
*/ */
@RunWith(AndroidJUnit4.class) @RunWith(AndroidJUnit4.class)
public final class FrameProcessorChainTest { public final class FrameProcessorChainTest {
private final AtomicReference<FrameProcessingException> frameProcessingException =
new AtomicReference<>();
@Test @Test
public void getOutputSize_noOperation_returnsInputSize() throws Exception { public void getOutputSize_noOperation_returnsInputSize() throws Exception {
...@@ -46,6 +49,7 @@ public final class FrameProcessorChainTest { ...@@ -46,6 +49,7 @@ public final class FrameProcessorChainTest {
Size outputSize = frameProcessorChain.getOutputSize(); Size outputSize = frameProcessorChain.getOutputSize();
assertThat(outputSize).isEqualTo(inputSize); assertThat(outputSize).isEqualTo(inputSize);
assertThat(frameProcessingException.get()).isNull();
} }
@Test @Test
...@@ -60,6 +64,7 @@ public final class FrameProcessorChainTest { ...@@ -60,6 +64,7 @@ public final class FrameProcessorChainTest {
Size outputSize = frameProcessorChain.getOutputSize(); Size outputSize = frameProcessorChain.getOutputSize();
assertThat(outputSize).isEqualTo(new Size(400, 100)); assertThat(outputSize).isEqualTo(new Size(400, 100));
assertThat(frameProcessingException.get()).isNull();
} }
@Test @Test
...@@ -74,6 +79,7 @@ public final class FrameProcessorChainTest { ...@@ -74,6 +79,7 @@ public final class FrameProcessorChainTest {
Size outputSize = frameProcessorChain.getOutputSize(); Size outputSize = frameProcessorChain.getOutputSize();
assertThat(outputSize).isEqualTo(new Size(200, 200)); assertThat(outputSize).isEqualTo(new Size(200, 200));
assertThat(frameProcessingException.get()).isNull();
} }
@Test @Test
...@@ -89,6 +95,7 @@ public final class FrameProcessorChainTest { ...@@ -89,6 +95,7 @@ public final class FrameProcessorChainTest {
Size frameProcessorChainOutputSize = frameProcessorChain.getOutputSize(); Size frameProcessorChainOutputSize = frameProcessorChain.getOutputSize();
assertThat(frameProcessorChainOutputSize).isEqualTo(frameProcessorOutputSize); assertThat(frameProcessorChainOutputSize).isEqualTo(frameProcessorOutputSize);
assertThat(frameProcessingException.get()).isNull();
} }
@Test @Test
...@@ -107,17 +114,19 @@ public final class FrameProcessorChainTest { ...@@ -107,17 +114,19 @@ public final class FrameProcessorChainTest {
Size frameProcessorChainOutputSize = frameProcessorChain.getOutputSize(); Size frameProcessorChainOutputSize = frameProcessorChain.getOutputSize();
assertThat(frameProcessorChainOutputSize).isEqualTo(outputSize3); assertThat(frameProcessorChainOutputSize).isEqualTo(outputSize3);
assertThat(frameProcessingException.get()).isNull();
} }
private static FrameProcessorChain createFrameProcessorChainWithFakeFrameProcessors( private FrameProcessorChain createFrameProcessorChainWithFakeFrameProcessors(
float pixelWidthHeightRatio, Size inputSize, List<Size> frameProcessorOutputSizes) float pixelWidthHeightRatio, Size inputSize, List<Size> frameProcessorOutputSizes)
throws TransformationException { throws FrameProcessingException {
ImmutableList.Builder<GlEffect> effects = new ImmutableList.Builder<>(); ImmutableList.Builder<GlEffect> effects = new ImmutableList.Builder<>();
for (Size element : frameProcessorOutputSizes) { for (Size element : frameProcessorOutputSizes) {
effects.add(() -> new FakeFrameProcessor(element)); effects.add(() -> new FakeFrameProcessor(element));
} }
return FrameProcessorChain.create( return FrameProcessorChain.create(
getApplicationContext(), getApplicationContext(),
/* listener= */ this.frameProcessingException::set,
pixelWidthHeightRatio, pixelWidthHeightRatio,
inputSize.getWidth(), inputSize.getWidth(),
inputSize.getHeight(), inputSize.getHeight(),
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExternalCopyFrameProcessor.java
...@@ -104,12 +104,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -104,12 +104,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
@Override @Override
public void drawFrame(long presentationTimeUs) { public void drawFrame(long presentationTimeUs) throws FrameProcessingException {
checkStateNotNull(glProgram); checkStateNotNull(glProgram);
try {
glProgram.use(); glProgram.use();
glProgram.bindAttributesAndUniforms(); glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad. // The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
GlUtil.checkGlError();
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e);
}
} }
@Override @Override
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameProcessingException.java 0 → 100644
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.transformer;
import com.google.android.exoplayer2.C;
/** Thrown when an exception occurs while applying effects to video frames. */
public final class FrameProcessingException extends Exception {
/**
* The microsecond timestamp of the frame being processed while the exception occurred or {@link
* C#TIME_UNSET} if unknown.
*/
public final long presentationTimeUs;
/**
* Creates an instance.
*
* @param message The detail message for this exception.
*/
public FrameProcessingException(String message) {
this(message, /* presentationTimeUs= */ C.TIME_UNSET);
}
/**
* Creates an instance.
*
* @param message The detail message for this exception.
* @param presentationTimeUs The timestamp of the frame for which the exception occurred.
*/
public FrameProcessingException(String message, long presentationTimeUs) {
super(message);
this.presentationTimeUs = presentationTimeUs;
}
/**
* Creates an instance.
*
* @param message The detail message for this exception.
* @param cause The cause of this exception.
*/
public FrameProcessingException(String message, Throwable cause) {
this(message, cause, /* presentationTimeUs= */ C.TIME_UNSET);
}
/**
* Creates an instance.
*
* @param message The detail message for this exception.
* @param cause The cause of this exception.
* @param presentationTimeUs The timestamp of the frame for which the exception occurred.
*/
public FrameProcessingException(String message, Throwable cause, long presentationTimeUs) {
super(message, cause);
this.presentationTimeUs = presentationTimeUs;
}
/**
* Creates an instance.
*
* @param cause The cause of this exception.
*/
public FrameProcessingException(Throwable cause) {
this(cause, /* presentationTimeUs= */ C.TIME_UNSET);
}
/**
* Creates an instance.
*
* @param cause The cause of this exception.
* @param presentationTimeUs The timestamp of the frame for which the exception occurred.
*/
public FrameProcessingException(Throwable cause, long presentationTimeUs) {
super(cause);
this.presentationTimeUs = presentationTimeUs;
}
}
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlFrameProcessor.java
...@@ -44,6 +44,7 @@ public interface GlFrameProcessor { ...@@ -44,6 +44,7 @@ public interface GlFrameProcessor {
* @param inputTexId Identifier of a 2D OpenGL texture. * @param inputTexId Identifier of a 2D OpenGL texture.
* @param inputWidth The input width, in pixels. * @param inputWidth The input width, in pixels.
* @param inputHeight The input height, in pixels. * @param inputHeight The input height, in pixels.
* @throws IOException If an error occurs while reading resources.
*/ */
void initialize(Context context, int inputTexId, int inputWidth, int inputHeight) void initialize(Context context, int inputTexId, int inputWidth, int inputHeight)
throws IOException; throws IOException;
...@@ -67,8 +68,9 @@ public interface GlFrameProcessor { ...@@ -67,8 +68,9 @@ public interface GlFrameProcessor {
* program's vertex attributes and uniforms, and issue a drawing command. * program's vertex attributes and uniforms, and issue a drawing command.
* *
* @param presentationTimeUs The presentation timestamp of the current frame, in microseconds. * @param presentationTimeUs The presentation timestamp of the current frame, in microseconds.
* @throws FrameProcessingException If an error occurs while processing or drawing the frame.
*/ */
void drawFrame(long presentationTimeUs); void drawFrame(long presentationTimeUs) throws FrameProcessingException;
/** Releases all resources. */ /** Releases all resources. */
void release(); void release();
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MatrixTransformationFrameProcessor.java
...@@ -95,7 +95,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -95,7 +95,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
@Override @Override
public void drawFrame(long presentationTimeUs) { public void drawFrame(long presentationTimeUs) throws FrameProcessingException {
try {
checkStateNotNull(glProgram).use(); checkStateNotNull(glProgram).use();
float[] transformationMatrix = matrixTransformation.getGlMatrixArray(presentationTimeUs); float[] transformationMatrix = matrixTransformation.getGlMatrixArray(presentationTimeUs);
checkState( checkState(
...@@ -105,6 +106,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -105,6 +106,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
// The four-vertex triangle strip forms a quad. // The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
GlUtil.checkGlError(); GlUtil.checkGlError();
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e);
}
} }
@Override @Override
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java
...@@ -719,6 +719,8 @@ public final class Transformer { ...@@ -719,6 +719,8 @@ public final class Transformer {
DEFAULT_BUFFER_FOR_PLAYBACK_MS / 10, DEFAULT_BUFFER_FOR_PLAYBACK_MS / 10,
DEFAULT_BUFFER_FOR_PLAYBACK_AFTER_REBUFFER_MS / 10) DEFAULT_BUFFER_FOR_PLAYBACK_AFTER_REBUFFER_MS / 10)
.build(); .build();
TransformerPlayerListener playerListener =
new TransformerPlayerListener(mediaItem, muxerWrapper, looper);
ExoPlayer.Builder playerBuilder = ExoPlayer.Builder playerBuilder =
new ExoPlayer.Builder( new ExoPlayer.Builder(
context, context,
...@@ -732,6 +734,7 @@ public final class Transformer { ...@@ -732,6 +734,7 @@ public final class Transformer {
encoderFactory, encoderFactory,
decoderFactory, decoderFactory,
new FallbackListener(mediaItem, listeners, transformationRequest), new FallbackListener(mediaItem, listeners, transformationRequest),
playerListener,
debugViewProvider)) debugViewProvider))
.setMediaSourceFactory(mediaSourceFactory) .setMediaSourceFactory(mediaSourceFactory)
.setTrackSelector(trackSelector) .setTrackSelector(trackSelector)
...@@ -746,7 +749,7 @@ public final class Transformer { ...@@ -746,7 +749,7 @@ public final class Transformer {
player = playerBuilder.build(); player = playerBuilder.build();
player.setMediaItem(mediaItem); player.setMediaItem(mediaItem);
player.addListener(new TransformerPlayerListener(mediaItem, muxerWrapper)); player.addListener(playerListener);
player.prepare(); player.prepare();
progressState = PROGRESS_STATE_WAITING_FOR_AVAILABILITY; progressState = PROGRESS_STATE_WAITING_FOR_AVAILABILITY;
...@@ -844,6 +847,7 @@ public final class Transformer { ...@@ -844,6 +847,7 @@ public final class Transformer {
private final Codec.EncoderFactory encoderFactory; private final Codec.EncoderFactory encoderFactory;
private final Codec.DecoderFactory decoderFactory; private final Codec.DecoderFactory decoderFactory;
private final FallbackListener fallbackListener; private final FallbackListener fallbackListener;
private final FrameProcessorChain.Listener frameProcessorChainListener;
private final Transformer.DebugViewProvider debugViewProvider; private final Transformer.DebugViewProvider debugViewProvider;
public TransformerRenderersFactory( public TransformerRenderersFactory(
...@@ -856,6 +860,7 @@ public final class Transformer { ...@@ -856,6 +860,7 @@ public final class Transformer {
Codec.EncoderFactory encoderFactory, Codec.EncoderFactory encoderFactory,
Codec.DecoderFactory decoderFactory, Codec.DecoderFactory decoderFactory,
FallbackListener fallbackListener, FallbackListener fallbackListener,
FrameProcessorChain.Listener frameProcessorChainListener,
Transformer.DebugViewProvider debugViewProvider) { Transformer.DebugViewProvider debugViewProvider) {
this.context = context; this.context = context;
this.muxerWrapper = muxerWrapper; this.muxerWrapper = muxerWrapper;
...@@ -866,6 +871,7 @@ public final class Transformer { ...@@ -866,6 +871,7 @@ public final class Transformer {
this.encoderFactory = encoderFactory; this.encoderFactory = encoderFactory;
this.decoderFactory = decoderFactory; this.decoderFactory = decoderFactory;
this.fallbackListener = fallbackListener; this.fallbackListener = fallbackListener;
this.frameProcessorChainListener = frameProcessorChainListener;
this.debugViewProvider = debugViewProvider; this.debugViewProvider = debugViewProvider;
mediaClock = new TransformerMediaClock(); mediaClock = new TransformerMediaClock();
} }
...@@ -902,6 +908,7 @@ public final class Transformer { ...@@ -902,6 +908,7 @@ public final class Transformer {
encoderFactory, encoderFactory,
decoderFactory, decoderFactory,
fallbackListener, fallbackListener,
frameProcessorChainListener,
debugViewProvider); debugViewProvider);
index++; index++;
} }
...@@ -909,14 +916,18 @@ public final class Transformer { ...@@ -909,14 +916,18 @@ public final class Transformer {
} }
} }
private final class TransformerPlayerListener implements Player.Listener { private final class TransformerPlayerListener
implements Player.Listener, FrameProcessorChain.Listener {
private final MediaItem mediaItem; private final MediaItem mediaItem;
private final MuxerWrapper muxerWrapper; private final MuxerWrapper muxerWrapper;
private final Handler handler;
public TransformerPlayerListener(MediaItem mediaItem, MuxerWrapper muxerWrapper) { public TransformerPlayerListener(
MediaItem mediaItem, MuxerWrapper muxerWrapper, Looper looper) {
this.mediaItem = mediaItem; this.mediaItem = mediaItem;
this.muxerWrapper = muxerWrapper; this.muxerWrapper = muxerWrapper;
handler = new Handler(looper);
} }
@Override @Override
...@@ -1011,5 +1022,14 @@ public final class Transformer { ...@@ -1011,5 +1022,14 @@ public final class Transformer {
} }
listeners.flushEvents(); listeners.flushEvents();
} }
@Override
public void onFrameProcessingError(FrameProcessingException exception) {
handler.post(
() ->
handleTransformationEnded(
TransformationException.createForFrameProcessorChain(
exception, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED)));
}
} }
} }
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java
...@@ -38,6 +38,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -38,6 +38,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private final ImmutableList<GlEffect> effects; private final ImmutableList<GlEffect> effects;
private final Codec.EncoderFactory encoderFactory; private final Codec.EncoderFactory encoderFactory;
private final Codec.DecoderFactory decoderFactory; private final Codec.DecoderFactory decoderFactory;
private final FrameProcessorChain.Listener frameProcessorChainListener;
private final Transformer.DebugViewProvider debugViewProvider; private final Transformer.DebugViewProvider debugViewProvider;
private final DecoderInputBuffer decoderInputBuffer; private final DecoderInputBuffer decoderInputBuffer;
...@@ -52,12 +53,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -52,12 +53,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
Codec.EncoderFactory encoderFactory, Codec.EncoderFactory encoderFactory,
Codec.DecoderFactory decoderFactory, Codec.DecoderFactory decoderFactory,
FallbackListener fallbackListener, FallbackListener fallbackListener,
FrameProcessorChain.Listener frameProcessorChainListener,
Transformer.DebugViewProvider debugViewProvider) { Transformer.DebugViewProvider debugViewProvider) {
super(C.TRACK_TYPE_VIDEO, muxerWrapper, mediaClock, transformationRequest, fallbackListener); super(C.TRACK_TYPE_VIDEO, muxerWrapper, mediaClock, transformationRequest, fallbackListener);
this.context = context; this.context = context;
this.effects = effects; this.effects = effects;
this.encoderFactory = encoderFactory; this.encoderFactory = encoderFactory;
this.decoderFactory = decoderFactory; this.decoderFactory = decoderFactory;
this.frameProcessorChainListener = frameProcessorChainListener;
this.debugViewProvider = debugViewProvider; this.debugViewProvider = debugViewProvider;
decoderInputBuffer = decoderInputBuffer =
new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
...@@ -95,6 +98,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -95,6 +98,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
encoderFactory, encoderFactory,
muxerWrapper.getSupportedSampleMimeTypes(getTrackType()), muxerWrapper.getSupportedSampleMimeTypes(getTrackType()),
fallbackListener, fallbackListener,
frameProcessorChainListener,
debugViewProvider); debugViewProvider);
} }
if (transformationRequest.flattenForSlowMotion) { if (transformationRequest.flattenForSlowMotion) {
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java
...@@ -55,6 +55,7 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -55,6 +55,7 @@ import org.checkerframework.dataflow.qual.Pure;
Codec.EncoderFactory encoderFactory, Codec.EncoderFactory encoderFactory,
List<String> allowedOutputMimeTypes, List<String> allowedOutputMimeTypes,
FallbackListener fallbackListener, FallbackListener fallbackListener,
FrameProcessorChain.Listener frameProcessorChainListener,
Transformer.DebugViewProvider debugViewProvider) Transformer.DebugViewProvider debugViewProvider)
throws TransformationException { throws TransformationException {
decoderInputBuffer = decoderInputBuffer =
...@@ -86,14 +87,20 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -86,14 +87,20 @@ import org.checkerframework.dataflow.qual.Pure;
EncoderCompatibilityTransformation encoderCompatibilityTransformation = EncoderCompatibilityTransformation encoderCompatibilityTransformation =
new EncoderCompatibilityTransformation(); new EncoderCompatibilityTransformation();
effectsListBuilder.add(encoderCompatibilityTransformation); effectsListBuilder.add(encoderCompatibilityTransformation);
try {
frameProcessorChain = frameProcessorChain =
FrameProcessorChain.create( FrameProcessorChain.create(
context, context,
frameProcessorChainListener,
inputFormat.pixelWidthHeightRatio, inputFormat.pixelWidthHeightRatio,
/* inputWidth= */ decodedWidth, /* inputWidth= */ decodedWidth,
/* inputHeight= */ decodedHeight, /* inputHeight= */ decodedHeight,
effectsListBuilder.build(), effectsListBuilder.build(),
transformationRequest.enableHdrEditing); transformationRequest.enableHdrEditing);
} catch (FrameProcessingException e) {
throw TransformationException.createForFrameProcessorChain(
e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
}
Size requestedEncoderSize = frameProcessorChain.getOutputSize(); Size requestedEncoderSize = frameProcessorChain.getOutputSize();
outputRotationDegrees = encoderCompatibilityTransformation.getOutputRotationDegrees(); outputRotationDegrees = encoderCompatibilityTransformation.getOutputRotationDegrees();
...@@ -146,7 +153,6 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -146,7 +153,6 @@ import org.checkerframework.dataflow.qual.Pure;
@Override @Override
public boolean processData() throws TransformationException { public boolean processData() throws TransformationException {
frameProcessorChain.getAndRethrowBackgroundExceptions();
if (frameProcessorChain.isEnded()) { if (frameProcessorChain.isEnded()) {
if (!signaledEndOfStreamToEncoder) { if (!signaledEndOfStreamToEncoder) {
encoder.signalEndOfInputStream(); encoder.signalEndOfInputStream();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment