Skip to content

SDK / exoplayer

Go to a project
Commit 63dcdf58 by hschlueter Committed by Ian Baker
Options

Add listener for FrameProcessingExceptions. 

This listener replaces
FrameProcessorChain#getAndRethrowBackgroundExceptions.
The listener uses a new exception type FrameProcessingException
separate from TransformationException as the frame processing
components will be made reusable outside of transformer soon.

PiperOrigin-RevId: 447455746
parent 1b15d5c3
Inline Side-by-side
Showing with 377 additions and 152 deletions
demos/transformer/src/main/java/androidx/media3/demo/transformer/BitmapOverlayFrameProcessor.java
...@@ -31,6 +31,7 @@ import android.util.Size; ...@@ -31,6 +31,7 @@ import android.util.Size;
import androidx.media3.common.C; import androidx.media3.common.C;
import androidx.media3.common.util.GlProgram; import androidx.media3.common.util.GlProgram;
import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.GlUtil;
import androidx.media3.transformer.FrameProcessingException;
import androidx.media3.transformer.GlFrameProcessor; import androidx.media3.transformer.GlFrameProcessor;
import java.io.IOException; import java.io.IOException;
import java.util.Locale; import java.util.Locale;
...@@ -116,28 +117,32 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -116,28 +117,32 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
@Override @Override
public void drawFrame(long presentationTimeUs) { public void drawFrame(long presentationTimeUs) throws FrameProcessingException {
checkStateNotNull(glProgram); try {
glProgram.use(); checkStateNotNull(glProgram).use();
// Draw to the canvas and store it in a texture. // Draw to the canvas and store it in a texture.
String text = String text =
String.format(Locale.US, "%.02f", presentationTimeUs / (float) C.MICROS_PER_SECOND); String.format(Locale.US, "%.02f", presentationTimeUs / (float) C.MICROS_PER_SECOND);
overlayBitmap.eraseColor(Color.TRANSPARENT); overlayBitmap.eraseColor(Color.TRANSPARENT);
overlayCanvas.drawBitmap(checkStateNotNull(logoBitmap), /* left= */ 3, /* top= */ 378, paint); overlayCanvas.drawBitmap(checkStateNotNull(logoBitmap), /* left= */ 3, /* top= */ 378, paint);
overlayCanvas.drawText(text, /* x= */ 160, /* y= */ 466, paint); overlayCanvas.drawText(text, /* x= */ 160, /* y= */ 466, paint);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, bitmapTexId); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, bitmapTexId);
GLUtils.texSubImage2D( GLUtils.texSubImage2D(
GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_2D,
/* level= */ 0, /* level= */ 0,
/* xoffset= */ 0, /* xoffset= */ 0,
/* yoffset= */ 0, /* yoffset= */ 0,
flipBitmapVertically(overlayBitmap)); flipBitmapVertically(overlayBitmap));
GlUtil.checkGlError(); GlUtil.checkGlError();
glProgram.bindAttributesAndUniforms(); glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad. // The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
GlUtil.checkGlError();
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e);
}
} }
@Override @Override
......
demos/transformer/src/main/java/androidx/media3/demo/transformer/PeriodicVignetteFrameProcessor.java
...@@ -23,6 +23,7 @@ import android.opengl.GLES20; ...@@ -23,6 +23,7 @@ import android.opengl.GLES20;
import android.util.Size; import android.util.Size;
import androidx.media3.common.util.GlProgram; import androidx.media3.common.util.GlProgram;
import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.GlUtil;
import androidx.media3.transformer.FrameProcessingException;
import androidx.media3.transformer.GlFrameProcessor; import androidx.media3.transformer.GlFrameProcessor;
import java.io.IOException; import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
...@@ -98,14 +99,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -98,14 +99,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
@Override @Override
public void drawFrame(long presentationTimeUs) { public void drawFrame(long presentationTimeUs) throws FrameProcessingException {
checkStateNotNull(glProgram).use(); try {
double theta = presentationTimeUs * 2 * Math.PI / DIMMING_PERIOD_US; checkStateNotNull(glProgram).use();
float innerRadius = minInnerRadius + deltaInnerRadius * (0.5f - 0.5f * (float) Math.cos(theta)); double theta = presentationTimeUs * 2 * Math.PI / DIMMING_PERIOD_US;
glProgram.setFloatsUniform("uInnerRadius", new float[] {innerRadius}); float innerRadius =
glProgram.bindAttributesAndUniforms(); minInnerRadius + deltaInnerRadius * (0.5f - 0.5f * (float) Math.cos(theta));
// The four-vertex triangle strip forms a quad. glProgram.setFloatsUniform("uInnerRadius", new float[] {innerRadius});
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e);
}
} }
@Override @Override
......
demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeFrameProcessor.java
...@@ -26,6 +26,7 @@ import androidx.media3.common.util.ConditionVariable; ...@@ -26,6 +26,7 @@ import androidx.media3.common.util.ConditionVariable;
import androidx.media3.common.util.GlProgram; import androidx.media3.common.util.GlProgram;
import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.LibraryLoader; import androidx.media3.common.util.LibraryLoader;
import androidx.media3.transformer.FrameProcessingException;
import androidx.media3.transformer.GlFrameProcessor; import androidx.media3.transformer.GlFrameProcessor;
import com.google.mediapipe.components.FrameProcessor; import com.google.mediapipe.components.FrameProcessor;
import com.google.mediapipe.framework.AppTextureFrame; import com.google.mediapipe.framework.AppTextureFrame;
...@@ -112,7 +113,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -112,7 +113,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
@Override @Override
public void drawFrame(long presentationTimeUs) { public void drawFrame(long presentationTimeUs) throws FrameProcessingException {
frameProcessorConditionVariable.close(); frameProcessorConditionVariable.close();
// Pass the input frame to MediaPipe. // Pass the input frame to MediaPipe.
...@@ -133,7 +134,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -133,7 +134,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
if (frameProcessorPendingError != null) { if (frameProcessorPendingError != null) {
throw new IllegalStateException(frameProcessorPendingError); throw new FrameProcessingException(frameProcessorPendingError);
} }
// Copy from MediaPipe's output texture to the current output. // Copy from MediaPipe's output texture to the current output.
...@@ -148,6 +149,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -148,6 +149,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
glProgram.bindAttributesAndUniforms(); glProgram.bindAttributesAndUniforms();
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
GlUtil.checkGlError(); GlUtil.checkGlError();
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e);
} finally { } finally {
checkStateNotNull(outputFrame).release(); checkStateNotNull(outputFrame).release();
} }
......
libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java
...@@ -50,6 +50,8 @@ public final class GlUtil { ...@@ -50,6 +50,8 @@ public final class GlUtil {
} }
} }
// TODO(b/231937416): Consider removing this flag, enabling assertions by default, and making
// GlException checked.
/** Whether to throw a {@link GlException} in case of an OpenGL error. */ /** Whether to throw a {@link GlException} in case of an OpenGL error. */
public static boolean glAssertionsEnabled = false; public static boolean glAssertionsEnabled = false;
......
libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java
...@@ -16,6 +16,7 @@ ...@@ -16,6 +16,7 @@
package androidx.media3.transformer; package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import static androidx.media3.transformer.BitmapTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE; import static androidx.media3.transformer.BitmapTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE;
import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertThat;
...@@ -36,6 +37,7 @@ import androidx.annotation.Nullable; ...@@ -36,6 +37,7 @@ import androidx.annotation.Nullable;
import androidx.media3.common.MimeTypes; import androidx.media3.common.MimeTypes;
import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.ext.junit.runners.AndroidJUnit4;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.concurrent.atomic.AtomicReference;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.junit.After; import org.junit.After;
import org.junit.Test; import org.junit.Test;
...@@ -78,6 +80,9 @@ public final class FrameProcessorChainPixelTest { ...@@ -78,6 +80,9 @@ public final class FrameProcessorChainPixelTest {
/** The ratio of width over height, for each pixel in a frame. */ /** The ratio of width over height, for each pixel in a frame. */
private static final float DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO = 1; private static final float DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO = 1;
private final AtomicReference<FrameProcessingException> frameProcessingException =
new AtomicReference<>();
private @MonotonicNonNull FrameProcessorChain frameProcessorChain; private @MonotonicNonNull FrameProcessorChain frameProcessorChain;
private @MonotonicNonNull ImageReader outputImageReader; private @MonotonicNonNull ImageReader outputImageReader;
private @MonotonicNonNull MediaFormat mediaFormat; private @MonotonicNonNull MediaFormat mediaFormat;
...@@ -229,6 +234,15 @@ public final class FrameProcessorChainPixelTest { ...@@ -229,6 +234,15 @@ public final class FrameProcessorChainPixelTest {
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
} }
@Test
public void processData_withFrameProcessingException_callsListener() throws Exception {
setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, ThrowingFrameProcessor::new);
Thread.sleep(FRAME_PROCESSING_WAIT_MS);
assertThat(frameProcessingException.get()).isNotNull();
}
/** /**
* Set up and prepare the first frame from an input video, as well as relevant test * Set up and prepare the first frame from an input video, as well as relevant test
* infrastructure. The frame will be sent towards the {@link FrameProcessorChain}, and may be * infrastructure. The frame will be sent towards the {@link FrameProcessorChain}, and may be
...@@ -258,6 +272,7 @@ public final class FrameProcessorChainPixelTest { ...@@ -258,6 +272,7 @@ public final class FrameProcessorChainPixelTest {
frameProcessorChain = frameProcessorChain =
FrameProcessorChain.create( FrameProcessorChain.create(
context, context,
/* listener= */ this.frameProcessingException::set,
pixelWidthHeightRatio, pixelWidthHeightRatio,
inputWidth, inputWidth,
inputHeight, inputHeight,
...@@ -321,11 +336,11 @@ public final class FrameProcessorChainPixelTest { ...@@ -321,11 +336,11 @@ public final class FrameProcessorChainPixelTest {
} }
} }
private Bitmap processFirstFrameAndEnd() throws InterruptedException, TransformationException { private Bitmap processFirstFrameAndEnd() throws InterruptedException {
checkNotNull(frameProcessorChain).signalEndOfInputStream(); checkNotNull(frameProcessorChain).signalEndOfInputStream();
Thread.sleep(FRAME_PROCESSING_WAIT_MS); Thread.sleep(FRAME_PROCESSING_WAIT_MS);
assertThat(frameProcessorChain.isEnded()).isTrue(); assertThat(frameProcessorChain.isEnded()).isTrue();
frameProcessorChain.getAndRethrowBackgroundExceptions(); assertThat(frameProcessingException.get()).isNull();
Image frameProcessorChainOutputImage = checkNotNull(outputImageReader).acquireLatestImage(); Image frameProcessorChainOutputImage = checkNotNull(outputImageReader).acquireLatestImage();
Bitmap actualBitmap = Bitmap actualBitmap =
...@@ -333,4 +348,27 @@ public final class FrameProcessorChainPixelTest { ...@@ -333,4 +348,27 @@ public final class FrameProcessorChainPixelTest {
frameProcessorChainOutputImage.close(); frameProcessorChainOutputImage.close();
return actualBitmap; return actualBitmap;
} }
private static class ThrowingFrameProcessor implements GlFrameProcessor {
private @MonotonicNonNull Size outputSize;
@Override
public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight) {
outputSize = new Size(inputWidth, inputHeight);
}
@Override
public Size getOutputSize() {
return checkStateNotNull(outputSize);
}
@Override
public void drawFrame(long presentationTimeUs) throws FrameProcessingException {
throw new FrameProcessingException("An exception occurred.");
}
@Override
public void release() {}
}
} }
libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java
...@@ -23,6 +23,7 @@ import android.util.Size; ...@@ -23,6 +23,7 @@ import android.util.Size;
import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import java.util.List; import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
...@@ -33,6 +34,8 @@ import org.junit.runner.RunWith; ...@@ -33,6 +34,8 @@ import org.junit.runner.RunWith;
*/ */
@RunWith(AndroidJUnit4.class) @RunWith(AndroidJUnit4.class)
public final class FrameProcessorChainTest { public final class FrameProcessorChainTest {
private final AtomicReference<FrameProcessingException> frameProcessingException =
new AtomicReference<>();
@Test @Test
public void getOutputSize_noOperation_returnsInputSize() throws Exception { public void getOutputSize_noOperation_returnsInputSize() throws Exception {
...@@ -46,6 +49,7 @@ public final class FrameProcessorChainTest { ...@@ -46,6 +49,7 @@ public final class FrameProcessorChainTest {
Size outputSize = frameProcessorChain.getOutputSize(); Size outputSize = frameProcessorChain.getOutputSize();
assertThat(outputSize).isEqualTo(inputSize); assertThat(outputSize).isEqualTo(inputSize);
assertThat(frameProcessingException.get()).isNull();
} }
@Test @Test
...@@ -60,6 +64,7 @@ public final class FrameProcessorChainTest { ...@@ -60,6 +64,7 @@ public final class FrameProcessorChainTest {
Size outputSize = frameProcessorChain.getOutputSize(); Size outputSize = frameProcessorChain.getOutputSize();
assertThat(outputSize).isEqualTo(new Size(400, 100)); assertThat(outputSize).isEqualTo(new Size(400, 100));
assertThat(frameProcessingException.get()).isNull();
} }
@Test @Test
...@@ -74,6 +79,7 @@ public final class FrameProcessorChainTest { ...@@ -74,6 +79,7 @@ public final class FrameProcessorChainTest {
Size outputSize = frameProcessorChain.getOutputSize(); Size outputSize = frameProcessorChain.getOutputSize();
assertThat(outputSize).isEqualTo(new Size(200, 200)); assertThat(outputSize).isEqualTo(new Size(200, 200));
assertThat(frameProcessingException.get()).isNull();
} }
@Test @Test
...@@ -89,6 +95,7 @@ public final class FrameProcessorChainTest { ...@@ -89,6 +95,7 @@ public final class FrameProcessorChainTest {
Size frameProcessorChainOutputSize = frameProcessorChain.getOutputSize(); Size frameProcessorChainOutputSize = frameProcessorChain.getOutputSize();
assertThat(frameProcessorChainOutputSize).isEqualTo(frameProcessorOutputSize); assertThat(frameProcessorChainOutputSize).isEqualTo(frameProcessorOutputSize);
assertThat(frameProcessingException.get()).isNull();
} }
@Test @Test
...@@ -107,17 +114,19 @@ public final class FrameProcessorChainTest { ...@@ -107,17 +114,19 @@ public final class FrameProcessorChainTest {
Size frameProcessorChainOutputSize = frameProcessorChain.getOutputSize(); Size frameProcessorChainOutputSize = frameProcessorChain.getOutputSize();
assertThat(frameProcessorChainOutputSize).isEqualTo(outputSize3); assertThat(frameProcessorChainOutputSize).isEqualTo(outputSize3);
assertThat(frameProcessingException.get()).isNull();
} }
private static FrameProcessorChain createFrameProcessorChainWithFakeFrameProcessors( private FrameProcessorChain createFrameProcessorChainWithFakeFrameProcessors(
float pixelWidthHeightRatio, Size inputSize, List<Size> frameProcessorOutputSizes) float pixelWidthHeightRatio, Size inputSize, List<Size> frameProcessorOutputSizes)
throws TransformationException { throws FrameProcessingException {
ImmutableList.Builder<GlEffect> effects = new ImmutableList.Builder<>(); ImmutableList.Builder<GlEffect> effects = new ImmutableList.Builder<>();
for (Size element : frameProcessorOutputSizes) { for (Size element : frameProcessorOutputSizes) {
effects.add(() -> new FakeFrameProcessor(element)); effects.add(() -> new FakeFrameProcessor(element));
} }
return FrameProcessorChain.create( return FrameProcessorChain.create(
getApplicationContext(), getApplicationContext(),
/* listener= */ this.frameProcessingException::set,
pixelWidthHeightRatio, pixelWidthHeightRatio,
inputSize.getWidth(), inputSize.getWidth(),
inputSize.getHeight(), inputSize.getHeight(),
......
libraries/transformer/src/main/java/androidx/media3/transformer/ExternalCopyFrameProcessor.java
...@@ -104,12 +104,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -104,12 +104,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
@Override @Override
public void drawFrame(long presentationTimeUs) { public void drawFrame(long presentationTimeUs) throws FrameProcessingException {
checkStateNotNull(glProgram); checkStateNotNull(glProgram);
glProgram.use(); try {
glProgram.bindAttributesAndUniforms(); glProgram.use();
// The four-vertex triangle strip forms a quad. glProgram.bindAttributesAndUniforms();
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); // The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
GlUtil.checkGlError();
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e);
}
} }
@Override @Override
......
libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessingException.java 0 → 100644
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer;
import androidx.media3.common.C;
import androidx.media3.common.util.UnstableApi;
/** Thrown when an exception occurs while applying effects to video frames. */
@UnstableApi
public final class FrameProcessingException extends Exception {
/**
* The microsecond timestamp of the frame being processed while the exception occurred or {@link
* C#TIME_UNSET} if unknown.
*/
public final long presentationTimeUs;
/**
* Creates an instance.
*
* @param message The detail message for this exception.
*/
public FrameProcessingException(String message) {
this(message, /* presentationTimeUs= */ C.TIME_UNSET);
}
/**
* Creates an instance.
*
* @param message The detail message for this exception.
* @param presentationTimeUs The timestamp of the frame for which the exception occurred.
*/
public FrameProcessingException(String message, long presentationTimeUs) {
super(message);
this.presentationTimeUs = presentationTimeUs;
}
/**
* Creates an instance.
*
* @param message The detail message for this exception.
* @param cause The cause of this exception.
*/
public FrameProcessingException(String message, Throwable cause) {
this(message, cause, /* presentationTimeUs= */ C.TIME_UNSET);
}
/**
* Creates an instance.
*
* @param message The detail message for this exception.
* @param cause The cause of this exception.
* @param presentationTimeUs The timestamp of the frame for which the exception occurred.
*/
public FrameProcessingException(String message, Throwable cause, long presentationTimeUs) {
super(message, cause);
this.presentationTimeUs = presentationTimeUs;
}
/**
* Creates an instance.
*
* @param cause The cause of this exception.
*/
public FrameProcessingException(Throwable cause) {
this(cause, /* presentationTimeUs= */ C.TIME_UNSET);
}
/**
* Creates an instance.
*
* @param cause The cause of this exception.
* @param presentationTimeUs The timestamp of the frame for which the exception occurred.
*/
public FrameProcessingException(Throwable cause, long presentationTimeUs) {
super(cause);
this.presentationTimeUs = presentationTimeUs;
}
}
libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java
...@@ -47,6 +47,7 @@ import java.util.concurrent.ExecutionException; ...@@ -47,6 +47,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.checker.nullness.qual.RequiresNonNull; import org.checkerframework.checker.nullness.qual.RequiresNonNull;
...@@ -68,9 +69,20 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -68,9 +69,20 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
} }
/** /**
* Listener for asynchronous frame processing events.
*
* <p>This listener is only called from the {@link FrameProcessorChain}'s background thread.
*/
public interface Listener {
/** Called when an exception occurs during asynchronous frame processing. */
void onFrameProcessingError(FrameProcessingException exception);
}
/**
* Creates a new instance. * Creates a new instance.
* *
* @param context A {@link Context}. * @param context A {@link Context}.
* @param listener A {@link Listener}.
* @param pixelWidthHeightRatio The ratio of width over height for each pixel. Pixels are expanded * @param pixelWidthHeightRatio The ratio of width over height for each pixel. Pixels are expanded
* by this ratio so that the output frame's pixels have a ratio of 1. * by this ratio so that the output frame's pixels have a ratio of 1.
* @param inputWidth The input frame width, in pixels. * @param inputWidth The input frame width, in pixels.
...@@ -78,17 +90,18 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -78,17 +90,18 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
* @param effects The {@link GlEffect GlEffects} to apply to each frame. * @param effects The {@link GlEffect GlEffects} to apply to each frame.
* @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal. * @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal.
* @return A new instance. * @return A new instance.
* @throws TransformationException If reading shader files fails, or an OpenGL error occurs while * @throws FrameProcessingException If reading shader files fails, or an OpenGL error occurs while
* creating and configuring the OpenGL components. * creating and configuring the OpenGL components.
*/ */
public static FrameProcessorChain create( public static FrameProcessorChain create(
Context context, Context context,
Listener listener,
float pixelWidthHeightRatio, float pixelWidthHeightRatio,
int inputWidth, int inputWidth,
int inputHeight, int inputHeight,
List<GlEffect> effects, List<GlEffect> effects,
boolean enableExperimentalHdrEditing) boolean enableExperimentalHdrEditing)
throws TransformationException { throws FrameProcessingException {
checkArgument(inputWidth > 0, "inputWidth must be positive"); checkArgument(inputWidth > 0, "inputWidth must be positive");
checkArgument(inputHeight > 0, "inputHeight must be positive"); checkArgument(inputHeight > 0, "inputHeight must be positive");
...@@ -100,6 +113,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -100,6 +113,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
() -> () ->
createOpenGlObjectsAndFrameProcessorChain( createOpenGlObjectsAndFrameProcessorChain(
context, context,
listener,
pixelWidthHeightRatio, pixelWidthHeightRatio,
inputWidth, inputWidth,
inputHeight, inputHeight,
...@@ -108,12 +122,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -108,12 +122,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
singleThreadExecutorService)) singleThreadExecutorService))
.get(); .get();
} catch (ExecutionException e) { } catch (ExecutionException e) {
throw TransformationException.createForFrameProcessorChain( throw new FrameProcessingException(e);
e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
} catch (InterruptedException e) { } catch (InterruptedException e) {
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
throw TransformationException.createForFrameProcessorChain( throw new FrameProcessingException(e);
e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
} }
} }
...@@ -127,6 +139,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -127,6 +139,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
@WorkerThread @WorkerThread
private static FrameProcessorChain createOpenGlObjectsAndFrameProcessorChain( private static FrameProcessorChain createOpenGlObjectsAndFrameProcessorChain(
Context context, Context context,
Listener listener,
float pixelWidthHeightRatio, float pixelWidthHeightRatio,
int inputWidth, int inputWidth,
int inputHeight, int inputHeight,
...@@ -177,6 +190,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -177,6 +190,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
inputExternalTexId, inputExternalTexId,
framebuffers, framebuffers,
frameProcessors, frameProcessors,
listener,
enableExperimentalHdrEditing); enableExperimentalHdrEditing);
} }
...@@ -241,6 +255,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -241,6 +255,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
*/ */
private final int[] framebuffers; private final int[] framebuffers;
private final Listener listener;
/**
* Prevents further frame processing tasks from being scheduled or executed after {@link
* #release()} is called or an exception occurred.
*/
private final AtomicBoolean stopProcessing;
private int outputWidth; private int outputWidth;
private int outputHeight; private int outputHeight;
/** /**
...@@ -258,8 +279,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -258,8 +279,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private @MonotonicNonNull EGLSurface debugPreviewEglSurface; private @MonotonicNonNull EGLSurface debugPreviewEglSurface;
private boolean inputStreamEnded; private boolean inputStreamEnded;
/** Prevents further frame processing tasks from being scheduled after {@link #release()}. */
private volatile boolean releaseRequested;
private FrameProcessorChain( private FrameProcessorChain(
EGLDisplay eglDisplay, EGLDisplay eglDisplay,
...@@ -268,6 +287,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -268,6 +287,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
int inputExternalTexId, int inputExternalTexId,
int[] framebuffers, int[] framebuffers,
ImmutableList<GlFrameProcessor> frameProcessors, ImmutableList<GlFrameProcessor> frameProcessors,
Listener listener,
boolean enableExperimentalHdrEditing) { boolean enableExperimentalHdrEditing) {
checkState(!frameProcessors.isEmpty()); checkState(!frameProcessors.isEmpty());
...@@ -276,6 +296,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -276,6 +296,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
this.singleThreadExecutorService = singleThreadExecutorService; this.singleThreadExecutorService = singleThreadExecutorService;
this.framebuffers = framebuffers; this.framebuffers = framebuffers;
this.frameProcessors = frameProcessors; this.frameProcessors = frameProcessors;
this.listener = listener;
this.stopProcessing = new AtomicBoolean();
this.enableExperimentalHdrEditing = enableExperimentalHdrEditing; this.enableExperimentalHdrEditing = enableExperimentalHdrEditing;
futures = new ConcurrentLinkedQueue<>(); futures = new ConcurrentLinkedQueue<>();
...@@ -331,7 +353,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -331,7 +353,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
inputSurfaceTexture.setOnFrameAvailableListener( inputSurfaceTexture.setOnFrameAvailableListener(
surfaceTexture -> { surfaceTexture -> {
if (releaseRequested) { if (stopProcessing.get()) {
// Frames can still become available after a transformation is cancelled but they can be // Frames can still become available after a transformation is cancelled but they can be
// ignored. // ignored.
return; return;
...@@ -339,7 +361,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -339,7 +361,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
try { try {
futures.add(singleThreadExecutorService.submit(this::processFrame)); futures.add(singleThreadExecutorService.submit(this::processFrame));
} catch (RejectedExecutionException e) { } catch (RejectedExecutionException e) {
if (!releaseRequested) { if (!stopProcessing.get()) {
throw e; throw e;
} }
} }
...@@ -371,28 +393,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -371,28 +393,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
return pendingFrameCount.get(); return pendingFrameCount.get();
} }
/**
* Checks whether any exceptions occurred during asynchronous frame processing and rethrows the
* first exception encountered.
*/
public void getAndRethrowBackgroundExceptions() throws TransformationException {
@Nullable Future<?> oldestGlProcessingFuture = futures.peek();
while (oldestGlProcessingFuture != null && oldestGlProcessingFuture.isDone()) {
futures.poll();
try {
oldestGlProcessingFuture.get();
} catch (ExecutionException e) {
throw TransformationException.createForFrameProcessorChain(
e, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw TransformationException.createForFrameProcessorChain(
e, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED);
}
oldestGlProcessingFuture = futures.peek();
}
}
/** Informs the {@code FrameProcessorChain} that no further input frames should be accepted. */ /** Informs the {@code FrameProcessorChain} that no further input frames should be accepted. */
public void signalEndOfInputStream() { public void signalEndOfInputStream() {
inputStreamEnded = true; inputStreamEnded = true;
...@@ -413,18 +413,12 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -413,18 +413,12 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
* <p>This method blocks until all OpenGL resources are released or releasing times out. * <p>This method blocks until all OpenGL resources are released or releasing times out.
*/ */
public void release() { public void release() {
releaseRequested = true; stopProcessing.set(true);
while (!futures.isEmpty()) { while (!futures.isEmpty()) {
checkNotNull(futures.poll()).cancel(/* mayInterruptIfRunning= */ true); checkNotNull(futures.poll()).cancel(/* mayInterruptIfRunning= */ true);
} }
futures.add( futures.add(
singleThreadExecutorService.submit( singleThreadExecutorService.submit(this::releaseFrameProcessorsAndDestroyGlContext));
() -> {
for (int i = 0; i < frameProcessors.size(); i++) {
frameProcessors.get(i).release();
}
GlUtil.destroyEglContext(eglDisplay, eglContext);
}));
if (inputSurfaceTexture != null) { if (inputSurfaceTexture != null) {
inputSurfaceTexture.release(); inputSurfaceTexture.release();
} }
...@@ -448,22 +442,26 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -448,22 +442,26 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
*/ */
@WorkerThread @WorkerThread
private void createOpenGlSurfaces(Surface outputSurface, @Nullable SurfaceView debugSurfaceView) { private void createOpenGlSurfaces(Surface outputSurface, @Nullable SurfaceView debugSurfaceView) {
checkState(Thread.currentThread().getName().equals(THREAD_NAME)); try {
checkStateNotNull(eglDisplay); checkState(Thread.currentThread().getName().equals(THREAD_NAME));
checkStateNotNull(eglDisplay);
if (enableExperimentalHdrEditing) {
// TODO(b/209404935): Don't assume BT.2020 PQ input/output. if (enableExperimentalHdrEditing) {
eglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, outputSurface); // TODO(b/209404935): Don't assume BT.2020 PQ input/output.
if (debugSurfaceView != null) { eglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, outputSurface);
debugPreviewEglSurface = if (debugSurfaceView != null) {
GlUtil.getEglSurfaceBt2020Pq(eglDisplay, checkNotNull(debugSurfaceView.getHolder())); debugPreviewEglSurface =
} GlUtil.getEglSurfaceBt2020Pq(eglDisplay, checkNotNull(debugSurfaceView.getHolder()));
} else { }
eglSurface = GlUtil.getEglSurface(eglDisplay, outputSurface); } else {
if (debugSurfaceView != null) { eglSurface = GlUtil.getEglSurface(eglDisplay, outputSurface);
debugPreviewEglSurface = if (debugSurfaceView != null) {
GlUtil.getEglSurface(eglDisplay, checkNotNull(debugSurfaceView.getHolder())); debugPreviewEglSurface =
GlUtil.getEglSurface(eglDisplay, checkNotNull(debugSurfaceView.getHolder()));
}
} }
} catch (RuntimeException e) {
listener.onFrameProcessingError(new FrameProcessingException(e));
} }
} }
...@@ -475,44 +473,58 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -475,44 +473,58 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
@WorkerThread @WorkerThread
@RequiresNonNull("inputSurfaceTexture") @RequiresNonNull("inputSurfaceTexture")
private void processFrame() { private void processFrame() {
checkState(Thread.currentThread().getName().equals(THREAD_NAME)); if (stopProcessing.get()) {
checkStateNotNull(eglSurface, "No output surface set."); return;
inputSurfaceTexture.updateTexImage();
long presentationTimeNs = inputSurfaceTexture.getTimestamp();
long presentationTimeUs = presentationTimeNs / 1000;
inputSurfaceTexture.getTransformMatrix(textureTransformMatrix);
((ExternalCopyFrameProcessor) frameProcessors.get(0))
.setTextureTransformMatrix(textureTransformMatrix);
for (int i = 0; i < frameProcessors.size() - 1; i++) {
Size intermediateSize = frameProcessors.get(i).getOutputSize();
GlUtil.focusFramebuffer(
eglDisplay,
eglContext,
eglSurface,
framebuffers[i],
intermediateSize.getWidth(),
intermediateSize.getHeight());
clearOutputFrame();
frameProcessors.get(i).drawFrame(presentationTimeUs);
} }
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight);
clearOutputFrame();
getLast(frameProcessors).drawFrame(presentationTimeUs);
EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, presentationTimeNs);
EGL14.eglSwapBuffers(eglDisplay, eglSurface);
if (debugPreviewEglSurface != null) { long presentationTimeUs = C.TIME_UNSET;
GlUtil.focusEglSurface( try {
eglDisplay, eglContext, debugPreviewEglSurface, debugPreviewWidth, debugPreviewHeight); checkState(Thread.currentThread().getName().equals(THREAD_NAME));
checkStateNotNull(eglSurface, "No output surface set.");
inputSurfaceTexture.updateTexImage();
long presentationTimeNs = inputSurfaceTexture.getTimestamp();
presentationTimeUs = presentationTimeNs / 1000;
inputSurfaceTexture.getTransformMatrix(textureTransformMatrix);
((ExternalCopyFrameProcessor) frameProcessors.get(0))
.setTextureTransformMatrix(textureTransformMatrix);
for (int i = 0; i < frameProcessors.size() - 1; i++) {
Size intermediateSize = frameProcessors.get(i).getOutputSize();
GlUtil.focusFramebuffer(
eglDisplay,
eglContext,
eglSurface,
framebuffers[i],
intermediateSize.getWidth(),
intermediateSize.getHeight());
clearOutputFrame();
frameProcessors.get(i).drawFrame(presentationTimeUs);
}
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight);
clearOutputFrame(); clearOutputFrame();
getLast(frameProcessors).drawFrame(presentationTimeUs); getLast(frameProcessors).drawFrame(presentationTimeUs);
EGL14.eglSwapBuffers(eglDisplay, debugPreviewEglSurface);
}
checkState(pendingFrameCount.getAndDecrement() > 0); EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, presentationTimeNs);
EGL14.eglSwapBuffers(eglDisplay, eglSurface);
if (debugPreviewEglSurface != null) {
GlUtil.focusEglSurface(
eglDisplay, eglContext, debugPreviewEglSurface, debugPreviewWidth, debugPreviewHeight);
clearOutputFrame();
getLast(frameProcessors).drawFrame(presentationTimeUs);
EGL14.eglSwapBuffers(eglDisplay, debugPreviewEglSurface);
}
checkState(pendingFrameCount.getAndDecrement() > 0);
} catch (FrameProcessingException | RuntimeException e) {
if (!stopProcessing.getAndSet(true)) {
listener.onFrameProcessingError(
e instanceof FrameProcessingException
? (FrameProcessingException) e
: new FrameProcessingException(e, presentationTimeUs));
}
}
} }
private static void clearOutputFrame() { private static void clearOutputFrame() {
...@@ -520,4 +532,21 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -520,4 +532,21 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
GlUtil.checkGlError(); GlUtil.checkGlError();
} }
/**
* Releases the {@link GlFrameProcessor GlFrameProcessors} and destroys the OpenGL context.
*
* <p>This method must be called on the {@linkplain #THREAD_NAME background thread}.
*/
@WorkerThread
private void releaseFrameProcessorsAndDestroyGlContext() {
try {
for (int i = 0; i < frameProcessors.size(); i++) {
frameProcessors.get(i).release();
}
GlUtil.destroyEglContext(eglDisplay, eglContext);
} catch (RuntimeException e) {
listener.onFrameProcessingError(new FrameProcessingException(e));
}
}
} }
libraries/transformer/src/main/java/androidx/media3/transformer/GlFrameProcessor.java
...@@ -46,6 +46,7 @@ public interface GlFrameProcessor { ...@@ -46,6 +46,7 @@ public interface GlFrameProcessor {
* @param inputTexId Identifier of a 2D OpenGL texture. * @param inputTexId Identifier of a 2D OpenGL texture.
* @param inputWidth The input width, in pixels. * @param inputWidth The input width, in pixels.
* @param inputHeight The input height, in pixels. * @param inputHeight The input height, in pixels.
* @throws IOException If an error occurs while reading resources.
*/ */
void initialize(Context context, int inputTexId, int inputWidth, int inputHeight) void initialize(Context context, int inputTexId, int inputWidth, int inputHeight)
throws IOException; throws IOException;
...@@ -69,8 +70,9 @@ public interface GlFrameProcessor { ...@@ -69,8 +70,9 @@ public interface GlFrameProcessor {
* program's vertex attributes and uniforms, and issue a drawing command. * program's vertex attributes and uniforms, and issue a drawing command.
* *
* @param presentationTimeUs The presentation timestamp of the current frame, in microseconds. * @param presentationTimeUs The presentation timestamp of the current frame, in microseconds.
* @throws FrameProcessingException If an error occurs while processing or drawing the frame.
*/ */
void drawFrame(long presentationTimeUs); void drawFrame(long presentationTimeUs) throws FrameProcessingException;
/** Releases all resources. */ /** Releases all resources. */
void release(); void release();
......
libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationFrameProcessor.java
...@@ -97,16 +97,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -97,16 +97,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
@Override @Override
public void drawFrame(long presentationTimeUs) { public void drawFrame(long presentationTimeUs) throws FrameProcessingException {
checkStateNotNull(glProgram).use(); try {
float[] transformationMatrix = matrixTransformation.getGlMatrixArray(presentationTimeUs); checkStateNotNull(glProgram).use();
checkState( float[] transformationMatrix = matrixTransformation.getGlMatrixArray(presentationTimeUs);
transformationMatrix.length == 16, "A 4x4 transformation matrix must have 16 elements"); checkState(
glProgram.setFloatsUniform("uTransformationMatrix", transformationMatrix); transformationMatrix.length == 16, "A 4x4 transformation matrix must have 16 elements");
glProgram.bindAttributesAndUniforms(); glProgram.setFloatsUniform("uTransformationMatrix", transformationMatrix);
// The four-vertex triangle strip forms a quad. glProgram.bindAttributesAndUniforms();
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); // The four-vertex triangle strip forms a quad.
GlUtil.checkGlError(); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
GlUtil.checkGlError();
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e);
}
} }
@Override @Override
......
libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
...@@ -721,6 +721,8 @@ public final class Transformer { ...@@ -721,6 +721,8 @@ public final class Transformer {
DEFAULT_BUFFER_FOR_PLAYBACK_MS / 10, DEFAULT_BUFFER_FOR_PLAYBACK_MS / 10,
DEFAULT_BUFFER_FOR_PLAYBACK_AFTER_REBUFFER_MS / 10) DEFAULT_BUFFER_FOR_PLAYBACK_AFTER_REBUFFER_MS / 10)
.build(); .build();
TransformerPlayerListener playerListener =
new TransformerPlayerListener(mediaItem, muxerWrapper, looper);
ExoPlayer.Builder playerBuilder = ExoPlayer.Builder playerBuilder =
new ExoPlayer.Builder( new ExoPlayer.Builder(
context, context,
...@@ -734,6 +736,7 @@ public final class Transformer { ...@@ -734,6 +736,7 @@ public final class Transformer {
encoderFactory, encoderFactory,
decoderFactory, decoderFactory,
new FallbackListener(mediaItem, listeners, transformationRequest), new FallbackListener(mediaItem, listeners, transformationRequest),
playerListener,
debugViewProvider)) debugViewProvider))
.setMediaSourceFactory(mediaSourceFactory) .setMediaSourceFactory(mediaSourceFactory)
.setTrackSelector(trackSelector) .setTrackSelector(trackSelector)
...@@ -748,7 +751,7 @@ public final class Transformer { ...@@ -748,7 +751,7 @@ public final class Transformer {
player = playerBuilder.build(); player = playerBuilder.build();
player.setMediaItem(mediaItem); player.setMediaItem(mediaItem);
player.addListener(new TransformerPlayerListener(mediaItem, muxerWrapper)); player.addListener(playerListener);
player.prepare(); player.prepare();
progressState = PROGRESS_STATE_WAITING_FOR_AVAILABILITY; progressState = PROGRESS_STATE_WAITING_FOR_AVAILABILITY;
...@@ -846,6 +849,7 @@ public final class Transformer { ...@@ -846,6 +849,7 @@ public final class Transformer {
private final Codec.EncoderFactory encoderFactory; private final Codec.EncoderFactory encoderFactory;
private final Codec.DecoderFactory decoderFactory; private final Codec.DecoderFactory decoderFactory;
private final FallbackListener fallbackListener; private final FallbackListener fallbackListener;
private final FrameProcessorChain.Listener frameProcessorChainListener;
private final Transformer.DebugViewProvider debugViewProvider; private final Transformer.DebugViewProvider debugViewProvider;
public TransformerRenderersFactory( public TransformerRenderersFactory(
...@@ -858,6 +862,7 @@ public final class Transformer { ...@@ -858,6 +862,7 @@ public final class Transformer {
Codec.EncoderFactory encoderFactory, Codec.EncoderFactory encoderFactory,
Codec.DecoderFactory decoderFactory, Codec.DecoderFactory decoderFactory,
FallbackListener fallbackListener, FallbackListener fallbackListener,
FrameProcessorChain.Listener frameProcessorChainListener,
Transformer.DebugViewProvider debugViewProvider) { Transformer.DebugViewProvider debugViewProvider) {
this.context = context; this.context = context;
this.muxerWrapper = muxerWrapper; this.muxerWrapper = muxerWrapper;
...@@ -868,6 +873,7 @@ public final class Transformer { ...@@ -868,6 +873,7 @@ public final class Transformer {
this.encoderFactory = encoderFactory; this.encoderFactory = encoderFactory;
this.decoderFactory = decoderFactory; this.decoderFactory = decoderFactory;
this.fallbackListener = fallbackListener; this.fallbackListener = fallbackListener;
this.frameProcessorChainListener = frameProcessorChainListener;
this.debugViewProvider = debugViewProvider; this.debugViewProvider = debugViewProvider;
mediaClock = new TransformerMediaClock(); mediaClock = new TransformerMediaClock();
} }
...@@ -904,6 +910,7 @@ public final class Transformer { ...@@ -904,6 +910,7 @@ public final class Transformer {
encoderFactory, encoderFactory,
decoderFactory, decoderFactory,
fallbackListener, fallbackListener,
frameProcessorChainListener,
debugViewProvider); debugViewProvider);
index++; index++;
} }
...@@ -911,14 +918,18 @@ public final class Transformer { ...@@ -911,14 +918,18 @@ public final class Transformer {
} }
} }
private final class TransformerPlayerListener implements Player.Listener { private final class TransformerPlayerListener
implements Player.Listener, FrameProcessorChain.Listener {
private final MediaItem mediaItem; private final MediaItem mediaItem;
private final MuxerWrapper muxerWrapper; private final MuxerWrapper muxerWrapper;
private final Handler handler;
public TransformerPlayerListener(MediaItem mediaItem, MuxerWrapper muxerWrapper) { public TransformerPlayerListener(
MediaItem mediaItem, MuxerWrapper muxerWrapper, Looper looper) {
this.mediaItem = mediaItem; this.mediaItem = mediaItem;
this.muxerWrapper = muxerWrapper; this.muxerWrapper = muxerWrapper;
handler = new Handler(looper);
} }
@Override @Override
...@@ -1013,5 +1024,14 @@ public final class Transformer { ...@@ -1013,5 +1024,14 @@ public final class Transformer {
} }
listeners.flushEvents(); listeners.flushEvents();
} }
@Override
public void onFrameProcessingError(FrameProcessingException exception) {
handler.post(
() ->
handleTransformationEnded(
TransformationException.createForFrameProcessorChain(
exception, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED)));
}
} }
} }
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java
...@@ -38,6 +38,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -38,6 +38,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private final ImmutableList<GlEffect> effects; private final ImmutableList<GlEffect> effects;
private final Codec.EncoderFactory encoderFactory; private final Codec.EncoderFactory encoderFactory;
private final Codec.DecoderFactory decoderFactory; private final Codec.DecoderFactory decoderFactory;
private final FrameProcessorChain.Listener frameProcessorChainListener;
private final Transformer.DebugViewProvider debugViewProvider; private final Transformer.DebugViewProvider debugViewProvider;
private final DecoderInputBuffer decoderInputBuffer; private final DecoderInputBuffer decoderInputBuffer;
...@@ -52,12 +53,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -52,12 +53,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
Codec.EncoderFactory encoderFactory, Codec.EncoderFactory encoderFactory,
Codec.DecoderFactory decoderFactory, Codec.DecoderFactory decoderFactory,
FallbackListener fallbackListener, FallbackListener fallbackListener,
FrameProcessorChain.Listener frameProcessorChainListener,
Transformer.DebugViewProvider debugViewProvider) { Transformer.DebugViewProvider debugViewProvider) {
super(C.TRACK_TYPE_VIDEO, muxerWrapper, mediaClock, transformationRequest, fallbackListener); super(C.TRACK_TYPE_VIDEO, muxerWrapper, mediaClock, transformationRequest, fallbackListener);
this.context = context; this.context = context;
this.effects = effects; this.effects = effects;
this.encoderFactory = encoderFactory; this.encoderFactory = encoderFactory;
this.decoderFactory = decoderFactory; this.decoderFactory = decoderFactory;
this.frameProcessorChainListener = frameProcessorChainListener;
this.debugViewProvider = debugViewProvider; this.debugViewProvider = debugViewProvider;
decoderInputBuffer = decoderInputBuffer =
new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
...@@ -95,6 +98,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -95,6 +98,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
encoderFactory, encoderFactory,
muxerWrapper.getSupportedSampleMimeTypes(getTrackType()), muxerWrapper.getSupportedSampleMimeTypes(getTrackType()),
fallbackListener, fallbackListener,
frameProcessorChainListener,
debugViewProvider); debugViewProvider);
} }
if (transformationRequest.flattenForSlowMotion) { if (transformationRequest.flattenForSlowMotion) {
......
libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
...@@ -55,6 +55,7 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -55,6 +55,7 @@ import org.checkerframework.dataflow.qual.Pure;
Codec.EncoderFactory encoderFactory, Codec.EncoderFactory encoderFactory,
List<String> allowedOutputMimeTypes, List<String> allowedOutputMimeTypes,
FallbackListener fallbackListener, FallbackListener fallbackListener,
FrameProcessorChain.Listener frameProcessorChainListener,
Transformer.DebugViewProvider debugViewProvider) Transformer.DebugViewProvider debugViewProvider)
throws TransformationException { throws TransformationException {
decoderInputBuffer = decoderInputBuffer =
...@@ -86,14 +87,20 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -86,14 +87,20 @@ import org.checkerframework.dataflow.qual.Pure;
EncoderCompatibilityTransformation encoderCompatibilityTransformation = EncoderCompatibilityTransformation encoderCompatibilityTransformation =
new EncoderCompatibilityTransformation(); new EncoderCompatibilityTransformation();
effectsListBuilder.add(encoderCompatibilityTransformation); effectsListBuilder.add(encoderCompatibilityTransformation);
frameProcessorChain = try {
FrameProcessorChain.create( frameProcessorChain =
context, FrameProcessorChain.create(
inputFormat.pixelWidthHeightRatio, context,
/* inputWidth= */ decodedWidth, frameProcessorChainListener,
/* inputHeight= */ decodedHeight, inputFormat.pixelWidthHeightRatio,
effectsListBuilder.build(), /* inputWidth= */ decodedWidth,
transformationRequest.enableHdrEditing); /* inputHeight= */ decodedHeight,
effectsListBuilder.build(),
transformationRequest.enableHdrEditing);
} catch (FrameProcessingException e) {
throw TransformationException.createForFrameProcessorChain(
e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
}
Size requestedEncoderSize = frameProcessorChain.getOutputSize(); Size requestedEncoderSize = frameProcessorChain.getOutputSize();
outputRotationDegrees = encoderCompatibilityTransformation.getOutputRotationDegrees(); outputRotationDegrees = encoderCompatibilityTransformation.getOutputRotationDegrees();
...@@ -146,7 +153,6 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -146,7 +153,6 @@ import org.checkerframework.dataflow.qual.Pure;
@Override @Override
public boolean processData() throws TransformationException { public boolean processData() throws TransformationException {
frameProcessorChain.getAndRethrowBackgroundExceptions();
if (frameProcessorChain.isEnded()) { if (frameProcessorChain.isEnded()) {
if (!signaledEndOfStreamToEncoder) { if (!signaledEndOfStreamToEncoder) {
encoder.signalEndOfInputStream(); encoder.signalEndOfInputStream();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment