Skip to content

SDK / exoplayer

Go to a project
Commit 0f5686fe by hschlueter Committed by Ian Baker
Options

Allow frame processors to be set on Transformer.Builder. 

Also make GlFrameProcessor, ScaleToFitFrameProcessor, and
AdvancedFrameProcessor public.

PiperOrigin-RevId: 437227388
parent d5056072
Inline Side-by-side
Showing with 61 additions and 8 deletions
libraries/transformer/src/main/java/androidx/media3/transformer/AdvancedFrameProcessor.java
...@@ -23,6 +23,7 @@ import android.opengl.GLES20; ...@@ -23,6 +23,7 @@ import android.opengl.GLES20;
import android.util.Size; import android.util.Size;
import androidx.media3.common.util.GlProgram; import androidx.media3.common.util.GlProgram;
import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.UnstableApi;
import java.io.IOException; import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
...@@ -32,7 +33,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -32,7 +33,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* ScaleToFitFrameProcessor}) are applied on the transformation. Width and height are not modified. * ScaleToFitFrameProcessor}) are applied on the transformation. Width and height are not modified.
* The background color will default to black. * The background color will default to black.
*/ */
/* package */ final class AdvancedFrameProcessor implements GlFrameProcessor { @UnstableApi
public final class AdvancedFrameProcessor implements GlFrameProcessor {
static { static {
GlUtil.glAssertionsEnabled = true; GlUtil.glAssertionsEnabled = true;
......
libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java
...@@ -152,7 +152,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -152,7 +152,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
} }
this.enableExperimentalHdrEditing = enableExperimentalHdrEditing; this.enableExperimentalHdrEditing = enableExperimentalHdrEditing;
this.frameProcessors = frameProcessors; this.frameProcessors = ImmutableList.copyOf(frameProcessors);
try { try {
eglDisplay = GlUtil.createEglDisplay(); eglDisplay = GlUtil.createEglDisplay();
......
libraries/transformer/src/main/java/androidx/media3/transformer/GlFrameProcessor.java
...@@ -16,6 +16,7 @@ ...@@ -16,6 +16,7 @@
package androidx.media3.transformer; package androidx.media3.transformer;
import android.util.Size; import android.util.Size;
import androidx.media3.common.util.UnstableApi;
import java.io.IOException; import java.io.IOException;
/** /**
...@@ -31,7 +32,8 @@ import java.io.IOException; ...@@ -31,7 +32,8 @@ import java.io.IOException;
* <li>{@link #release()}, upon conclusion of processing. * <li>{@link #release()}, upon conclusion of processing.
* </ol> * </ol>
*/ */
/* package */ interface GlFrameProcessor { @UnstableApi
public interface GlFrameProcessor {
// TODO(b/214975934): Investigate whether all configuration can be moved to initialize by // TODO(b/214975934): Investigate whether all configuration can be moved to initialize by
// using a placeholder surface until the encoder surface is known. If so, convert // using a placeholder surface until the encoder surface is known. If so, convert
// configureOutputSize to a simple getter. // configureOutputSize to a simple getter.
......
libraries/transformer/src/main/java/androidx/media3/transformer/PresentationFrameProcessor.java
...@@ -24,12 +24,14 @@ import android.util.Size; ...@@ -24,12 +24,14 @@ import android.util.Size;
import androidx.media3.common.C; import androidx.media3.common.C;
import androidx.media3.common.Format; import androidx.media3.common.Format;
import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.UnstableApi;
import java.io.IOException; import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** Controls how a frame is viewed, by changing resolution. */ /** Controls how a frame is viewed, by changing resolution. */
// TODO(b/213190310): Implement crop, aspect ratio changes, etc. // TODO(b/213190310): Implement crop, aspect ratio changes, etc.
/* package */ final class PresentationFrameProcessor implements GlFrameProcessor { @UnstableApi
public final class PresentationFrameProcessor implements GlFrameProcessor {
/** A builder for {@link PresentationFrameProcessor} instances. */ /** A builder for {@link PresentationFrameProcessor} instances. */
public static final class Builder { public static final class Builder {
......
libraries/transformer/src/main/java/androidx/media3/transformer/ScaleToFitFrameProcessor.java
...@@ -24,6 +24,7 @@ import android.graphics.Matrix; ...@@ -24,6 +24,7 @@ import android.graphics.Matrix;
import android.util.Size; import android.util.Size;
import androidx.media3.common.C; import androidx.media3.common.C;
import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.UnstableApi;
import java.io.IOException; import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
...@@ -32,7 +33,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -32,7 +33,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* preserved, potentially changing the width and height of the frame by scaling dimensions to fit. * preserved, potentially changing the width and height of the frame by scaling dimensions to fit.
* The background color will default to black. * The background color will default to black.
*/ */
/* package */ final class ScaleToFitFrameProcessor implements GlFrameProcessor { @UnstableApi
public final class ScaleToFitFrameProcessor implements GlFrameProcessor {
/** A builder for {@link ScaleToFitFrameProcessor} instances. */ /** A builder for {@link ScaleToFitFrameProcessor} instances. */
public static final class Builder { public static final class Builder {
......
libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
...@@ -59,11 +59,13 @@ import androidx.media3.exoplayer.trackselection.DefaultTrackSelector; ...@@ -59,11 +59,13 @@ import androidx.media3.exoplayer.trackselection.DefaultTrackSelector;
import androidx.media3.exoplayer.video.VideoRendererEventListener; import androidx.media3.exoplayer.video.VideoRendererEventListener;
import androidx.media3.extractor.DefaultExtractorsFactory; import androidx.media3.extractor.DefaultExtractorsFactory;
import androidx.media3.extractor.mp4.Mp4Extractor; import androidx.media3.extractor.mp4.Mp4Extractor;
import com.google.common.collect.ImmutableList;
import java.io.IOException; import java.io.IOException;
import java.lang.annotation.Documented; import java.lang.annotation.Documented;
import java.lang.annotation.Retention; import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy; import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target; import java.lang.annotation.Target;
import java.util.List;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** /**
...@@ -102,6 +104,7 @@ public final class Transformer { ...@@ -102,6 +104,7 @@ public final class Transformer {
private boolean removeVideo; private boolean removeVideo;
private String containerMimeType; private String containerMimeType;
private TransformationRequest transformationRequest; private TransformationRequest transformationRequest;
private ImmutableList<GlFrameProcessor> frameProcessors;
private ListenerSet<Transformer.Listener> listeners; private ListenerSet<Transformer.Listener> listeners;
private DebugViewProvider debugViewProvider; private DebugViewProvider debugViewProvider;
private Looper looper; private Looper looper;
...@@ -121,6 +124,7 @@ public final class Transformer { ...@@ -121,6 +124,7 @@ public final class Transformer {
debugViewProvider = DebugViewProvider.NONE; debugViewProvider = DebugViewProvider.NONE;
containerMimeType = MimeTypes.VIDEO_MP4; containerMimeType = MimeTypes.VIDEO_MP4;
transformationRequest = new TransformationRequest.Builder().build(); transformationRequest = new TransformationRequest.Builder().build();
frameProcessors = ImmutableList.of();
} }
/** /**
...@@ -137,7 +141,8 @@ public final class Transformer { ...@@ -137,7 +141,8 @@ public final class Transformer {
encoderFactory = Codec.EncoderFactory.DEFAULT; encoderFactory = Codec.EncoderFactory.DEFAULT;
debugViewProvider = DebugViewProvider.NONE; debugViewProvider = DebugViewProvider.NONE;
containerMimeType = MimeTypes.VIDEO_MP4; containerMimeType = MimeTypes.VIDEO_MP4;
this.transformationRequest = new TransformationRequest.Builder().build(); transformationRequest = new TransformationRequest.Builder().build();
frameProcessors = ImmutableList.of();
} }
/** Creates a builder with the values of the provided {@link Transformer}. */ /** Creates a builder with the values of the provided {@link Transformer}. */
...@@ -149,6 +154,7 @@ public final class Transformer { ...@@ -149,6 +154,7 @@ public final class Transformer {
this.removeVideo = transformer.removeVideo; this.removeVideo = transformer.removeVideo;
this.containerMimeType = transformer.containerMimeType; this.containerMimeType = transformer.containerMimeType;
this.transformationRequest = transformer.transformationRequest; this.transformationRequest = transformer.transformationRequest;
this.frameProcessors = transformer.frameProcessors;
this.listeners = transformer.listeners; this.listeners = transformer.listeners;
this.looper = transformer.looper; this.looper = transformer.looper;
this.encoderFactory = transformer.encoderFactory; this.encoderFactory = transformer.encoderFactory;
...@@ -181,6 +187,24 @@ public final class Transformer { ...@@ -181,6 +187,24 @@ public final class Transformer {
} }
/** /**
* Sets the {@linkplain GlFrameProcessor frame processors} to apply to each frame.
*
* <p>The {@linkplain GlFrameProcessor frame processors} are applied before any {@linkplain
* TransformationRequest.Builder#setScale(float, float) scale}, {@linkplain
* TransformationRequest.Builder#setRotationDegrees(float) rotation}, or {@linkplain
* TransformationRequest.Builder#setResolution(int) resolution} changes specified in the {@link
* #setTransformationRequest(TransformationRequest) TransformationRequest} but after {@linkplain
* TransformationRequest.Builder#setFlattenForSlowMotion(boolean) slow-motion flattening}.
*
* @param frameProcessors The {@linkplain GlFrameProcessor frame processors}.
* @return This builder.
*/
public Builder setFrameProcessors(List<GlFrameProcessor> frameProcessors) {
this.frameProcessors = ImmutableList.copyOf(frameProcessors);
return this;
}
/**
* Sets the {@link MediaSource.Factory} to be used to retrieve the inputs to transform. * Sets the {@link MediaSource.Factory} to be used to retrieve the inputs to transform.
* *
* <p>The default value is a {@link DefaultMediaSourceFactory} built with the context provided * <p>The default value is a {@link DefaultMediaSourceFactory} built with the context provided
...@@ -408,6 +432,7 @@ public final class Transformer { ...@@ -408,6 +432,7 @@ public final class Transformer {
removeVideo, removeVideo,
containerMimeType, containerMimeType,
transformationRequest, transformationRequest,
frameProcessors,
listeners, listeners,
looper, looper,
clock, clock,
...@@ -529,6 +554,7 @@ public final class Transformer { ...@@ -529,6 +554,7 @@ public final class Transformer {
private final boolean removeVideo; private final boolean removeVideo;
private final String containerMimeType; private final String containerMimeType;
private final TransformationRequest transformationRequest; private final TransformationRequest transformationRequest;
private final ImmutableList<GlFrameProcessor> frameProcessors;
private final Looper looper; private final Looper looper;
private final Clock clock; private final Clock clock;
private final Codec.EncoderFactory encoderFactory; private final Codec.EncoderFactory encoderFactory;
...@@ -549,6 +575,7 @@ public final class Transformer { ...@@ -549,6 +575,7 @@ public final class Transformer {
boolean removeVideo, boolean removeVideo,
String containerMimeType, String containerMimeType,
TransformationRequest transformationRequest, TransformationRequest transformationRequest,
ImmutableList<GlFrameProcessor> frameProcessors,
ListenerSet<Transformer.Listener> listeners, ListenerSet<Transformer.Listener> listeners,
Looper looper, Looper looper,
Clock clock, Clock clock,
...@@ -563,6 +590,7 @@ public final class Transformer { ...@@ -563,6 +590,7 @@ public final class Transformer {
this.removeVideo = removeVideo; this.removeVideo = removeVideo;
this.containerMimeType = containerMimeType; this.containerMimeType = containerMimeType;
this.transformationRequest = transformationRequest; this.transformationRequest = transformationRequest;
this.frameProcessors = frameProcessors;
this.listeners = listeners; this.listeners = listeners;
this.looper = looper; this.looper = looper;
this.clock = clock; this.clock = clock;
...@@ -703,6 +731,7 @@ public final class Transformer { ...@@ -703,6 +731,7 @@ public final class Transformer {
removeAudio, removeAudio,
removeVideo, removeVideo,
transformationRequest, transformationRequest,
frameProcessors,
encoderFactory, encoderFactory,
decoderFactory, decoderFactory,
new FallbackListener(mediaItem, listeners, transformationRequest), new FallbackListener(mediaItem, listeners, transformationRequest),
...@@ -814,6 +843,7 @@ public final class Transformer { ...@@ -814,6 +843,7 @@ public final class Transformer {
private final boolean removeAudio; private final boolean removeAudio;
private final boolean removeVideo; private final boolean removeVideo;
private final TransformationRequest transformationRequest; private final TransformationRequest transformationRequest;
private final ImmutableList<GlFrameProcessor> frameProcessors;
private final Codec.EncoderFactory encoderFactory; private final Codec.EncoderFactory encoderFactory;
private final Codec.DecoderFactory decoderFactory; private final Codec.DecoderFactory decoderFactory;
private final FallbackListener fallbackListener; private final FallbackListener fallbackListener;
...@@ -825,6 +855,7 @@ public final class Transformer { ...@@ -825,6 +855,7 @@ public final class Transformer {
boolean removeAudio, boolean removeAudio,
boolean removeVideo, boolean removeVideo,
TransformationRequest transformationRequest, TransformationRequest transformationRequest,
ImmutableList<GlFrameProcessor> frameProcessors,
Codec.EncoderFactory encoderFactory, Codec.EncoderFactory encoderFactory,
Codec.DecoderFactory decoderFactory, Codec.DecoderFactory decoderFactory,
FallbackListener fallbackListener, FallbackListener fallbackListener,
...@@ -834,6 +865,7 @@ public final class Transformer { ...@@ -834,6 +865,7 @@ public final class Transformer {
this.removeAudio = removeAudio; this.removeAudio = removeAudio;
this.removeVideo = removeVideo; this.removeVideo = removeVideo;
this.transformationRequest = transformationRequest; this.transformationRequest = transformationRequest;
this.frameProcessors = frameProcessors;
this.encoderFactory = encoderFactory; this.encoderFactory = encoderFactory;
this.decoderFactory = decoderFactory; this.decoderFactory = decoderFactory;
this.fallbackListener = fallbackListener; this.fallbackListener = fallbackListener;
...@@ -869,6 +901,7 @@ public final class Transformer { ...@@ -869,6 +901,7 @@ public final class Transformer {
muxerWrapper, muxerWrapper,
mediaClock, mediaClock,
transformationRequest, transformationRequest,
frameProcessors,
encoderFactory, encoderFactory,
decoderFactory, decoderFactory,
fallbackListener, fallbackListener,
......
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java
...@@ -25,6 +25,7 @@ import androidx.media3.common.Format; ...@@ -25,6 +25,7 @@ import androidx.media3.common.Format;
import androidx.media3.decoder.DecoderInputBuffer; import androidx.media3.decoder.DecoderInputBuffer;
import androidx.media3.exoplayer.FormatHolder; import androidx.media3.exoplayer.FormatHolder;
import androidx.media3.exoplayer.source.SampleStream.ReadDataResult; import androidx.media3.exoplayer.source.SampleStream.ReadDataResult;
import com.google.common.collect.ImmutableList;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.checker.nullness.qual.RequiresNonNull; import org.checkerframework.checker.nullness.qual.RequiresNonNull;
...@@ -34,6 +35,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -34,6 +35,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private static final String TAG = "TVideoRenderer"; private static final String TAG = "TVideoRenderer";
private final Context context; private final Context context;
private final ImmutableList<GlFrameProcessor> frameProcessors;
private final Codec.EncoderFactory encoderFactory; private final Codec.EncoderFactory encoderFactory;
private final Codec.DecoderFactory decoderFactory; private final Codec.DecoderFactory decoderFactory;
private final Transformer.DebugViewProvider debugViewProvider; private final Transformer.DebugViewProvider debugViewProvider;
...@@ -46,12 +48,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -46,12 +48,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
MuxerWrapper muxerWrapper, MuxerWrapper muxerWrapper,
TransformerMediaClock mediaClock, TransformerMediaClock mediaClock,
TransformationRequest transformationRequest, TransformationRequest transformationRequest,
ImmutableList<GlFrameProcessor> frameProcessors,
Codec.EncoderFactory encoderFactory, Codec.EncoderFactory encoderFactory,
Codec.DecoderFactory decoderFactory, Codec.DecoderFactory decoderFactory,
FallbackListener fallbackListener, FallbackListener fallbackListener,
Transformer.DebugViewProvider debugViewProvider) { Transformer.DebugViewProvider debugViewProvider) {
super(C.TRACK_TYPE_VIDEO, muxerWrapper, mediaClock, transformationRequest, fallbackListener); super(C.TRACK_TYPE_VIDEO, muxerWrapper, mediaClock, transformationRequest, fallbackListener);
this.context = context; this.context = context;
this.frameProcessors = frameProcessors;
this.encoderFactory = encoderFactory; this.encoderFactory = encoderFactory;
this.decoderFactory = decoderFactory; this.decoderFactory = decoderFactory;
this.debugViewProvider = debugViewProvider; this.debugViewProvider = debugViewProvider;
...@@ -86,6 +90,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -86,6 +90,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
context, context,
inputFormat, inputFormat,
transformationRequest, transformationRequest,
frameProcessors,
decoderFactory, decoderFactory,
encoderFactory, encoderFactory,
muxerWrapper.getSupportedSampleMimeTypes(getTrackType()), muxerWrapper.getSupportedSampleMimeTypes(getTrackType()),
...@@ -126,6 +131,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -126,6 +131,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
&& transformationRequest.outputHeight != inputFormat.height) { && transformationRequest.outputHeight != inputFormat.height) {
return false; return false;
} }
if (!frameProcessors.isEmpty()) {
return false;
}
return true; return true;
} }
......
libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
...@@ -52,6 +52,7 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -52,6 +52,7 @@ import org.checkerframework.dataflow.qual.Pure;
Context context, Context context,
Format inputFormat, Format inputFormat,
TransformationRequest transformationRequest, TransformationRequest transformationRequest,
ImmutableList<GlFrameProcessor> frameProcessors,
Codec.DecoderFactory decoderFactory, Codec.DecoderFactory decoderFactory,
Codec.EncoderFactory encoderFactory, Codec.EncoderFactory encoderFactory,
List<String> allowedOutputMimeTypes, List<String> allowedOutputMimeTypes,
...@@ -69,7 +70,6 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -69,7 +70,6 @@ import org.checkerframework.dataflow.qual.Pure;
int decodedHeight = int decodedHeight =
(inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width; (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width;
// TODO(b/214975934): Allow a list of frame processors to be passed into the sample pipeline.
// TODO(b/213190310): Don't create a ScaleToFitFrameProcessor if scale and rotation are unset. // TODO(b/213190310): Don't create a ScaleToFitFrameProcessor if scale and rotation are unset.
ScaleToFitFrameProcessor scaleToFitFrameProcessor = ScaleToFitFrameProcessor scaleToFitFrameProcessor =
new ScaleToFitFrameProcessor.Builder(context) new ScaleToFitFrameProcessor.Builder(context)
...@@ -86,7 +86,11 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -86,7 +86,11 @@ import org.checkerframework.dataflow.qual.Pure;
inputFormat.pixelWidthHeightRatio, inputFormat.pixelWidthHeightRatio,
/* inputWidth= */ decodedWidth, /* inputWidth= */ decodedWidth,
/* inputHeight= */ decodedHeight, /* inputHeight= */ decodedHeight,
ImmutableList.of(scaleToFitFrameProcessor, presentationFrameProcessor), new ImmutableList.Builder<GlFrameProcessor>()
.addAll(frameProcessors)
.add(scaleToFitFrameProcessor)
.add(presentationFrameProcessor)
.build(),
transformationRequest.enableHdrEditing); transformationRequest.enableHdrEditing);
Size requestedEncoderSize = frameProcessorChain.getOutputSize(); Size requestedEncoderSize = frameProcessorChain.getOutputSize();
outputRotationDegrees = presentationFrameProcessor.getOutputRotationDegrees(); outputRotationDegrees = presentationFrameProcessor.getOutputRotationDegrees();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment