Skip to content

SDK / exoplayer

Go to a project
Commit ecbe9a50 by Googler Committed by Marc Baechinger
Options

Allow FrameProcessor.Factory to be set on Transformer.Builder. 

Extract a FrameProcessor.Factory interface from GlEffectsFrameProcessor
and allow it to  be customized using a setter on Transformer.Builder.

PiperOrigin-RevId: 463433438
parent 3c2b54b5
Inline Side-by-side
Showing with 88 additions and 16 deletions
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/GlEffectsFrameProcessorPixelTest.java
...@@ -356,7 +356,8 @@ public final class GlEffectsFrameProcessorPixelTest { ...@@ -356,7 +356,8 @@ public final class GlEffectsFrameProcessorPixelTest {
int inputHeight = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT); int inputHeight = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT);
glEffectsFrameProcessor = glEffectsFrameProcessor =
checkNotNull( checkNotNull(
GlEffectsFrameProcessor.create( new GlEffectsFrameProcessor.Factory()
.create(
context, context,
new FrameProcessor.Listener() { new FrameProcessor.Listener() {
@Override @Override
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameProcessor.java
...@@ -15,11 +15,49 @@ ...@@ -15,11 +15,49 @@
*/ */
package com.google.android.exoplayer2.transformer; package com.google.android.exoplayer2.transformer;
import android.content.Context;
import android.view.Surface; import android.view.Surface;
import androidx.annotation.Nullable; import androidx.annotation.Nullable;
import java.util.List;
/**
* Interface for a frame processor that applies changes to individual video frames.
*
* <p>The changes are specified by {@link GlEffect} instances passed to the {@link Factory}.
*
* <p>The frame processor manages its input {@link Surface} which can be accessed via {@link
* #getInputSurface()}. The output {@link Surface} must be set by the caller using {@link
* #setOutputSurfaceInfo(SurfaceInfo)}.
*
* <p>The caller must {@linkplain #registerInputFrame() register} input frames before rendering them
* to the input {@link Surface}.
*/
public interface FrameProcessor {
// TODO(b/227625423): Allow effects to be replaced.
/** A factory for {@link FrameProcessor} instances. */
interface Factory {
/**
* Creates a new {@link FrameProcessor} instance.
*
* @param context A {@link Context}.
* @param listener A {@link Listener}.
* @param effects The {@link GlEffect} instances to apply to each frame.
* @param debugViewProvider A {@link DebugViewProvider}.
* @param useHdr Whether to process the input as an HDR signal.
* @return A new instance.
* @throws FrameProcessingException If a problem occurs while creating the {@link
* FrameProcessor}.
*/
FrameProcessor create(
Context context,
FrameProcessor.Listener listener,
List<GlEffect> effects,
DebugViewProvider debugViewProvider,
boolean useHdr)
throws FrameProcessingException;
}
/** Interface for a frame processor that applies changes to individual video frames. */
/* package */ interface FrameProcessor {
/** /**
* Listener for asynchronous frame processing events. * Listener for asynchronous frame processing events.
* *
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlEffectsFrameProcessor.java
...@@ -42,24 +42,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -42,24 +42,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* A {@link FrameProcessor} implementation that applies {@link GlEffect} instances using OpenGL on a * A {@link FrameProcessor} implementation that applies {@link GlEffect} instances using OpenGL on a
* background thread. * background thread.
*/ */
/* package */ final class GlEffectsFrameProcessor implements FrameProcessor { public final class GlEffectsFrameProcessor implements FrameProcessor {
// TODO(b/227625423): Replace factory method with setters once output surface and effects can be // TODO(b/227625423): Replace factory method with setters once output surface and effects can be
// replaced. // replaced.
/** A factory for {@link GlEffectsFrameProcessor} instances. */
public static class Factory implements FrameProcessor.Factory {
/** /**
* Creates a new instance. * {@inheritDoc}
* *
* @param context A {@link Context}. * <p>Using HDR requires the {@code EXT_YUV_target} OpenGL extension.
* @param listener A {@link Listener}.
* @param effects The {@link GlEffect GlEffects} to apply to each frame.
* @param debugViewProvider A {@link DebugViewProvider}.
* @param useHdr Whether to process the input as an HDR signal. Using HDR requires the {@code
* EXT_YUV_target} OpenGL extension.
* @return A new instance.
* @throws FrameProcessingException If reading shader files fails, or an OpenGL error occurs while
* creating and configuring the OpenGL components.
*/ */
public static GlEffectsFrameProcessor create( @Override
public GlEffectsFrameProcessor create(
Context context, Context context,
FrameProcessor.Listener listener, FrameProcessor.Listener listener,
List<GlEffect> effects, List<GlEffect> effects,
...@@ -89,6 +84,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -89,6 +84,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
throw new FrameProcessingException(e); throw new FrameProcessingException(e);
} }
} }
}
/** /**
* Creates the OpenGL context, surfaces, textures, and framebuffers, initializes {@link * Creates the OpenGL context, surfaces, textures, and framebuffers, initializes {@link
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java
...@@ -102,6 +102,7 @@ public final class Transformer { ...@@ -102,6 +102,7 @@ public final class Transformer {
private String containerMimeType; private String containerMimeType;
private TransformationRequest transformationRequest; private TransformationRequest transformationRequest;
private ImmutableList<GlEffect> videoEffects; private ImmutableList<GlEffect> videoEffects;
private FrameProcessor.Factory frameProcessorFactory;
private ListenerSet<Transformer.Listener> listeners; private ListenerSet<Transformer.Listener> listeners;
private DebugViewProvider debugViewProvider; private DebugViewProvider debugViewProvider;
private Looper looper; private Looper looper;
...@@ -126,6 +127,7 @@ public final class Transformer { ...@@ -126,6 +127,7 @@ public final class Transformer {
containerMimeType = MimeTypes.VIDEO_MP4; containerMimeType = MimeTypes.VIDEO_MP4;
transformationRequest = new TransformationRequest.Builder().build(); transformationRequest = new TransformationRequest.Builder().build();
videoEffects = ImmutableList.of(); videoEffects = ImmutableList.of();
frameProcessorFactory = new GlEffectsFrameProcessor.Factory();
} }
/** Creates a builder with the values of the provided {@link Transformer}. */ /** Creates a builder with the values of the provided {@link Transformer}. */
...@@ -138,6 +140,7 @@ public final class Transformer { ...@@ -138,6 +140,7 @@ public final class Transformer {
this.containerMimeType = transformer.containerMimeType; this.containerMimeType = transformer.containerMimeType;
this.transformationRequest = transformer.transformationRequest; this.transformationRequest = transformer.transformationRequest;
this.videoEffects = transformer.videoEffects; this.videoEffects = transformer.videoEffects;
this.frameProcessorFactory = transformer.frameProcessorFactory;
this.listeners = transformer.listeners; this.listeners = transformer.listeners;
this.looper = transformer.looper; this.looper = transformer.looper;
this.encoderFactory = transformer.encoderFactory; this.encoderFactory = transformer.encoderFactory;
...@@ -182,6 +185,26 @@ public final class Transformer { ...@@ -182,6 +185,26 @@ public final class Transformer {
} }
/** /**
* Sets the {@link FrameProcessor.Factory} for the {@link FrameProcessor} to use when applying
* {@linkplain GlEffect effects} to the video frames.
*
* <p>This factory will be used to create the {@link FrameProcessor} used for applying the
* {@link GlEffect} instances passed to {@link #setVideoEffects(List<GlEffect>)} and any
* additional {@link GlMatrixTransformation} instances derived from the {@link
* TransformationRequest} set using {@link #setTransformationRequest(TransformationRequest)}.
*
* <p>The default is {@link GlEffectsFrameProcessor.Factory}.
*
* @param frameProcessorFactory The {@link FrameProcessor.Factory} to use.
* @return This builder.
*/
@CanIgnoreReturnValue
public Builder setFrameProcessorFactory(FrameProcessor.Factory frameProcessorFactory) {
this.frameProcessorFactory = frameProcessorFactory;
return this;
}
/**
* Sets the {@link MediaSource.Factory} to be used to retrieve the inputs to transform. * Sets the {@link MediaSource.Factory} to be used to retrieve the inputs to transform.
* *
* <p>The default value is a {@link DefaultMediaSourceFactory} built with the context provided * <p>The default value is a {@link DefaultMediaSourceFactory} built with the context provided
...@@ -438,6 +461,7 @@ public final class Transformer { ...@@ -438,6 +461,7 @@ public final class Transformer {
containerMimeType, containerMimeType,
transformationRequest, transformationRequest,
videoEffects, videoEffects,
frameProcessorFactory,
listeners, listeners,
looper, looper,
clock, clock,
...@@ -546,6 +570,7 @@ public final class Transformer { ...@@ -546,6 +570,7 @@ public final class Transformer {
private final String containerMimeType; private final String containerMimeType;
private final TransformationRequest transformationRequest; private final TransformationRequest transformationRequest;
private final ImmutableList<GlEffect> videoEffects; private final ImmutableList<GlEffect> videoEffects;
private final FrameProcessor.Factory frameProcessorFactory;
private final Looper looper; private final Looper looper;
private final Clock clock; private final Clock clock;
private final DebugViewProvider debugViewProvider; private final DebugViewProvider debugViewProvider;
...@@ -567,6 +592,7 @@ public final class Transformer { ...@@ -567,6 +592,7 @@ public final class Transformer {
String containerMimeType, String containerMimeType,
TransformationRequest transformationRequest, TransformationRequest transformationRequest,
ImmutableList<GlEffect> videoEffects, ImmutableList<GlEffect> videoEffects,
FrameProcessor.Factory frameProcessorFactory,
ListenerSet<Transformer.Listener> listeners, ListenerSet<Transformer.Listener> listeners,
Looper looper, Looper looper,
Clock clock, Clock clock,
...@@ -582,6 +608,7 @@ public final class Transformer { ...@@ -582,6 +608,7 @@ public final class Transformer {
this.containerMimeType = containerMimeType; this.containerMimeType = containerMimeType;
this.transformationRequest = transformationRequest; this.transformationRequest = transformationRequest;
this.videoEffects = videoEffects; this.videoEffects = videoEffects;
this.frameProcessorFactory = frameProcessorFactory;
this.listeners = listeners; this.listeners = listeners;
this.looper = looper; this.looper = looper;
this.clock = clock; this.clock = clock;
...@@ -731,6 +758,7 @@ public final class Transformer { ...@@ -731,6 +758,7 @@ public final class Transformer {
transformationRequest, transformationRequest,
mediaItem.clippingConfiguration.startsAtKeyFrame, mediaItem.clippingConfiguration.startsAtKeyFrame,
videoEffects, videoEffects,
frameProcessorFactory,
encoderFactory, encoderFactory,
decoderFactory, decoderFactory,
new FallbackListener(mediaItem, listeners, transformationRequest), new FallbackListener(mediaItem, listeners, transformationRequest),
...@@ -845,6 +873,7 @@ public final class Transformer { ...@@ -845,6 +873,7 @@ public final class Transformer {
private final TransformationRequest transformationRequest; private final TransformationRequest transformationRequest;
private final boolean clippingStartsAtKeyFrame; private final boolean clippingStartsAtKeyFrame;
private final ImmutableList<GlEffect> videoEffects; private final ImmutableList<GlEffect> videoEffects;
private final FrameProcessor.Factory frameProcessorFactory;
private final Codec.EncoderFactory encoderFactory; private final Codec.EncoderFactory encoderFactory;
private final Codec.DecoderFactory decoderFactory; private final Codec.DecoderFactory decoderFactory;
private final FallbackListener fallbackListener; private final FallbackListener fallbackListener;
...@@ -859,6 +888,7 @@ public final class Transformer { ...@@ -859,6 +888,7 @@ public final class Transformer {
TransformationRequest transformationRequest, TransformationRequest transformationRequest,
boolean clippingStartsAtKeyFrame, boolean clippingStartsAtKeyFrame,
ImmutableList<GlEffect> videoEffects, ImmutableList<GlEffect> videoEffects,
FrameProcessor.Factory frameProcessorFactory,
Codec.EncoderFactory encoderFactory, Codec.EncoderFactory encoderFactory,
Codec.DecoderFactory decoderFactory, Codec.DecoderFactory decoderFactory,
FallbackListener fallbackListener, FallbackListener fallbackListener,
...@@ -871,6 +901,7 @@ public final class Transformer { ...@@ -871,6 +901,7 @@ public final class Transformer {
this.transformationRequest = transformationRequest; this.transformationRequest = transformationRequest;
this.clippingStartsAtKeyFrame = clippingStartsAtKeyFrame; this.clippingStartsAtKeyFrame = clippingStartsAtKeyFrame;
this.videoEffects = videoEffects; this.videoEffects = videoEffects;
this.frameProcessorFactory = frameProcessorFactory;
this.encoderFactory = encoderFactory; this.encoderFactory = encoderFactory;
this.decoderFactory = decoderFactory; this.decoderFactory = decoderFactory;
this.fallbackListener = fallbackListener; this.fallbackListener = fallbackListener;
...@@ -910,6 +941,7 @@ public final class Transformer { ...@@ -910,6 +941,7 @@ public final class Transformer {
transformationRequest, transformationRequest,
clippingStartsAtKeyFrame, clippingStartsAtKeyFrame,
videoEffects, videoEffects,
frameProcessorFactory,
encoderFactory, encoderFactory,
decoderFactory, decoderFactory,
asyncErrorListener, asyncErrorListener,
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java
...@@ -39,6 +39,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -39,6 +39,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private final Context context; private final Context context;
private final boolean clippingStartsAtKeyFrame; private final boolean clippingStartsAtKeyFrame;
private final ImmutableList<GlEffect> effects; private final ImmutableList<GlEffect> effects;
private final FrameProcessor.Factory frameProcessorFactory;
private final Codec.EncoderFactory encoderFactory; private final Codec.EncoderFactory encoderFactory;
private final Codec.DecoderFactory decoderFactory; private final Codec.DecoderFactory decoderFactory;
private final DebugViewProvider debugViewProvider; private final DebugViewProvider debugViewProvider;
...@@ -53,6 +54,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -53,6 +54,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
TransformationRequest transformationRequest, TransformationRequest transformationRequest,
boolean clippingStartsAtKeyFrame, boolean clippingStartsAtKeyFrame,
ImmutableList<GlEffect> effects, ImmutableList<GlEffect> effects,
FrameProcessor.Factory frameProcessorFactory,
Codec.EncoderFactory encoderFactory, Codec.EncoderFactory encoderFactory,
Codec.DecoderFactory decoderFactory, Codec.DecoderFactory decoderFactory,
Transformer.AsyncErrorListener asyncErrorListener, Transformer.AsyncErrorListener asyncErrorListener,
...@@ -68,6 +70,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -68,6 +70,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
this.context = context; this.context = context;
this.clippingStartsAtKeyFrame = clippingStartsAtKeyFrame; this.clippingStartsAtKeyFrame = clippingStartsAtKeyFrame;
this.effects = effects; this.effects = effects;
this.frameProcessorFactory = frameProcessorFactory;
this.encoderFactory = encoderFactory; this.encoderFactory = encoderFactory;
this.decoderFactory = decoderFactory; this.decoderFactory = decoderFactory;
this.debugViewProvider = debugViewProvider; this.debugViewProvider = debugViewProvider;
...@@ -113,6 +116,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -113,6 +116,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
streamOffsetUs, streamOffsetUs,
transformationRequest, transformationRequest,
effects, effects,
frameProcessorFactory,
decoderFactory, decoderFactory,
encoderFactory, encoderFactory,
muxerWrapper.getSupportedSampleMimeTypes(getTrackType()), muxerWrapper.getSupportedSampleMimeTypes(getTrackType()),
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java
...@@ -58,6 +58,7 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -58,6 +58,7 @@ import org.checkerframework.dataflow.qual.Pure;
long streamOffsetUs, long streamOffsetUs,
TransformationRequest transformationRequest, TransformationRequest transformationRequest,
ImmutableList<GlEffect> effects, ImmutableList<GlEffect> effects,
FrameProcessor.Factory frameProcessorFactory,
Codec.DecoderFactory decoderFactory, Codec.DecoderFactory decoderFactory,
Codec.EncoderFactory encoderFactory, Codec.EncoderFactory encoderFactory,
List<String> allowedOutputMimeTypes, List<String> allowedOutputMimeTypes,
...@@ -102,7 +103,7 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -102,7 +103,7 @@ import org.checkerframework.dataflow.qual.Pure;
try { try {
frameProcessor = frameProcessor =
GlEffectsFrameProcessor.create( frameProcessorFactory.create(
context, context,
new FrameProcessor.Listener() { new FrameProcessor.Listener() {
@Override @Override
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment