Skip to content

SDK / exoplayer

Go to a project
Commit b410a922 by hschlueter Committed by Ian Baker
Options

Introduce GlEffect interface for effect specification. 

PiperOrigin-RevId: 446143537
parent 90ce9a67
Inline Side-by-side
Showing with 159 additions and 121 deletions
demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java
......@@ -58,7 +58,7 @@ public final class ConfigurationActivity extends AppCompatActivity {
public static final String ENABLE_FALLBACK = "enable_fallback";
public static final String ENABLE_REQUEST_SDR_TONE_MAPPING = "enable_request_sdr_tone_mapping";
public static final String ENABLE_HDR_EDITING = "enable_hdr_editing";
public static final String DEMO_FRAME_PROCESSORS_SELECTIONS = "demo_frame_processors_selections";
public static final String DEMO_EFFECTS_SELECTIONS = "demo_effects_selections";
public static final String PERIODIC_VIGNETTE_CENTER_X = "periodic_vignette_center_x";
public static final String PERIODIC_VIGNETTE_CENTER_Y = "periodic_vignette_center_y";
public static final String PERIODIC_VIGNETTE_INNER_RADIUS = "periodic_vignette_inner_radius";
......@@ -91,7 +91,7 @@ public final class ConfigurationActivity extends AppCompatActivity {
"DASH stream with non-square pixels",
"MP4 with HDR (HDR10) H265 video (encoding may fail)",
};
private static final String[] DEMO_FRAME_PROCESSORS = {
private static final String[] DEMO_EFFECTS = {
"Dizzy crop", "Periodic vignette", "3D spin", "Overlay logo & timer", "Zoom in start"
};
private static final int PERIODIC_VIGNETTE_INDEX = 1;
......@@ -111,8 +111,8 @@ public final class ConfigurationActivity extends AppCompatActivity {
private @MonotonicNonNull CheckBox enableFallbackCheckBox;
private @MonotonicNonNull CheckBox enableRequestSdrToneMappingCheckBox;
private @MonotonicNonNull CheckBox enableHdrEditingCheckBox;
private @MonotonicNonNull Button selectDemoFrameProcessorsButton;
private boolean @MonotonicNonNull [] demoFrameProcessorsSelections;
private @MonotonicNonNull Button selectDemoEffectsButton;
private boolean @MonotonicNonNull [] demoEffectsSelections;
private int inputUriPosition;
private float periodicVignetteCenterX;
private float periodicVignetteCenterY;
......@@ -187,9 +187,9 @@ public final class ConfigurationActivity extends AppCompatActivity {
findViewById(R.id.request_sdr_tone_mapping).setEnabled(isRequestSdrToneMappingSupported());
enableHdrEditingCheckBox = findViewById(R.id.hdr_editing_checkbox);
demoFrameProcessorsSelections = new boolean[DEMO_FRAME_PROCESSORS.length];
selectDemoFrameProcessorsButton = findViewById(R.id.select_demo_frameprocessors_button);
selectDemoFrameProcessorsButton.setOnClickListener(this::selectFrameProcessors);
demoEffectsSelections = new boolean[DEMO_EFFECTS.length];
selectDemoEffectsButton = findViewById(R.id.select_demo_effects_button);
selectDemoEffectsButton.setOnClickListener(this::selectDemoEffects);
}
@Override
......@@ -220,7 +220,7 @@ public final class ConfigurationActivity extends AppCompatActivity {
"enableFallbackCheckBox",
"enableRequestSdrToneMappingCheckBox",
"enableHdrEditingCheckBox",
"demoFrameProcessorsSelections"
"demoEffectsSelections"
})
private void startTransformation(View view) {
Intent transformerIntent = new Intent(/* packageContext= */ this, TransformerActivity.class);
......@@ -255,7 +255,7 @@ public final class ConfigurationActivity extends AppCompatActivity {
bundle.putBoolean(
ENABLE_REQUEST_SDR_TONE_MAPPING, enableRequestSdrToneMappingCheckBox.isChecked());
bundle.putBoolean(ENABLE_HDR_EDITING, enableHdrEditingCheckBox.isChecked());
bundle.putBooleanArray(DEMO_FRAME_PROCESSORS_SELECTIONS, demoFrameProcessorsSelections);
bundle.putBooleanArray(DEMO_EFFECTS_SELECTIONS, demoEffectsSelections);
bundle.putFloat(PERIODIC_VIGNETTE_CENTER_X, periodicVignetteCenterX);
bundle.putFloat(PERIODIC_VIGNETTE_CENTER_Y, periodicVignetteCenterY);
bundle.putFloat(PERIODIC_VIGNETTE_INNER_RADIUS, periodicVignetteInnerRadius);
......@@ -278,13 +278,11 @@ public final class ConfigurationActivity extends AppCompatActivity {
.show();
}
private void selectFrameProcessors(View view) {
private void selectDemoEffects(View view) {
new AlertDialog.Builder(/* context= */ this)
.setTitle(R.string.select_demo_frameprocessors)
.setTitle(R.string.select_demo_effects)
.setMultiChoiceItems(
DEMO_FRAME_PROCESSORS,
checkNotNull(demoFrameProcessorsSelections),
this::selectFrameProcessor)
DEMO_EFFECTS, checkNotNull(demoEffectsSelections), this::selectDemoEffect)
.setPositiveButton(android.R.string.ok, /* listener= */ null)
.create()
.show();
......@@ -296,9 +294,9 @@ public final class ConfigurationActivity extends AppCompatActivity {
selectedFileTextView.setText(URI_DESCRIPTIONS[inputUriPosition]);
}
@RequiresNonNull("demoFrameProcessorsSelections")
private void selectFrameProcessor(DialogInterface dialog, int which, boolean isChecked) {
demoFrameProcessorsSelections[which] = isChecked;
@RequiresNonNull("demoEffectsSelections")
private void selectDemoEffect(DialogInterface dialog, int which, boolean isChecked) {
demoEffectsSelections[which] = isChecked;
if (!isChecked || which != PERIODIC_VIGNETTE_INDEX) {
return;
}
......@@ -337,7 +335,7 @@ public final class ConfigurationActivity extends AppCompatActivity {
"rotateSpinner",
"enableRequestSdrToneMappingCheckBox",
"enableHdrEditingCheckBox",
"selectDemoFrameProcessorsButton"
"selectDemoEffectsButton"
})
private void onRemoveAudio(View view) {
if (((CheckBox) view).isChecked()) {
......@@ -357,7 +355,7 @@ public final class ConfigurationActivity extends AppCompatActivity {
"rotateSpinner",
"enableRequestSdrToneMappingCheckBox",
"enableHdrEditingCheckBox",
"selectDemoFrameProcessorsButton"
"selectDemoEffectsButton"
})
private void onRemoveVideo(View view) {
if (((CheckBox) view).isChecked()) {
......@@ -376,7 +374,7 @@ public final class ConfigurationActivity extends AppCompatActivity {
"rotateSpinner",
"enableRequestSdrToneMappingCheckBox",
"enableHdrEditingCheckBox",
"selectDemoFrameProcessorsButton"
"selectDemoEffectsButton"
})
private void enableTrackSpecificOptions(boolean isAudioEnabled, boolean isVideoEnabled) {
audioMimeSpinner.setEnabled(isAudioEnabled);
......@@ -387,7 +385,7 @@ public final class ConfigurationActivity extends AppCompatActivity {
enableRequestSdrToneMappingCheckBox.setEnabled(
isRequestSdrToneMappingSupported() && isVideoEnabled);
enableHdrEditingCheckBox.setEnabled(isVideoEnabled);
selectDemoFrameProcessorsButton.setEnabled(isVideoEnabled);
selectDemoEffectsButton.setEnabled(isVideoEnabled);
findViewById(R.id.audio_mime_text_view).setEnabled(isAudioEnabled);
findViewById(R.id.video_mime_text_view).setEnabled(isVideoEnabled);
......
demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java
......@@ -40,7 +40,7 @@ import androidx.media3.exoplayer.ExoPlayer;
import androidx.media3.exoplayer.util.DebugTextViewHelper;
import androidx.media3.transformer.DefaultEncoderFactory;
import androidx.media3.transformer.EncoderSelector;
import androidx.media3.transformer.GlFrameProcessor;
import androidx.media3.transformer.GlEffect;
import androidx.media3.transformer.ProgressHolder;
import androidx.media3.transformer.TransformationException;
import androidx.media3.transformer.TransformationRequest;
......@@ -240,35 +240,36 @@ public final class TransformerActivity extends AppCompatActivity {
EncoderSelector.DEFAULT,
/* enableFallback= */ bundle.getBoolean(ConfigurationActivity.ENABLE_FALLBACK)));
ImmutableList.Builder<GlFrameProcessor> frameProcessors = new ImmutableList.Builder<>();
ImmutableList.Builder<GlEffect> effects = new ImmutableList.Builder<>();
@Nullable
boolean[] selectedFrameProcessors =
bundle.getBooleanArray(ConfigurationActivity.DEMO_FRAME_PROCESSORS_SELECTIONS);
if (selectedFrameProcessors != null) {
if (selectedFrameProcessors[0]) {
frameProcessors.add(AdvancedFrameProcessorFactory.createDizzyCropFrameProcessor());
boolean[] selectedEffects =
bundle.getBooleanArray(ConfigurationActivity.DEMO_EFFECTS_SELECTIONS);
if (selectedEffects != null) {
if (selectedEffects[0]) {
effects.add(AdvancedFrameProcessorFactory::createDizzyCropFrameProcessor);
}
if (selectedFrameProcessors[1]) {
frameProcessors.add(
new PeriodicVignetteFrameProcessor(
bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_X),
bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_Y),
/* minInnerRadius= */ bundle.getFloat(
ConfigurationActivity.PERIODIC_VIGNETTE_INNER_RADIUS),
/* maxInnerRadius= */ bundle.getFloat(
ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS),
bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS)));
if (selectedEffects[1]) {
effects.add(
() ->
new PeriodicVignetteFrameProcessor(
bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_X),
bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_Y),
/* minInnerRadius= */ bundle.getFloat(
ConfigurationActivity.PERIODIC_VIGNETTE_INNER_RADIUS),
/* maxInnerRadius= */ bundle.getFloat(
ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS),
bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS)));
}
if (selectedFrameProcessors[2]) {
frameProcessors.add(AdvancedFrameProcessorFactory.createSpin3dFrameProcessor());
if (selectedEffects[2]) {
effects.add(AdvancedFrameProcessorFactory::createSpin3dFrameProcessor);
}
if (selectedFrameProcessors[3]) {
frameProcessors.add(new BitmapOverlayFrameProcessor());
if (selectedEffects[3]) {
effects.add(BitmapOverlayFrameProcessor::new);
}
if (selectedFrameProcessors[4]) {
frameProcessors.add(AdvancedFrameProcessorFactory.createZoomInTransitionFrameProcessor());
if (selectedEffects[4]) {
effects.add(AdvancedFrameProcessorFactory::createZoomInTransitionFrameProcessor);
}
transformerBuilder.setFrameProcessors(frameProcessors.build());
transformerBuilder.setVideoFrameEffects(effects.build());
}
}
return transformerBuilder
......
demos/transformer/src/main/res/layout/configuration_activity.xml
......@@ -64,7 +64,7 @@
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintTop_toBottomOf="@+id/selected_file_text_view"
app:layout_constraintBottom_toTopOf="@+id/select_demo_frameprocessors_button">
app:layout_constraintBottom_toTopOf="@+id/select_demo_effects_button">
<TableLayout
android:layout_width="fill_parent"
android:layout_height="wrap_content"
......@@ -192,13 +192,13 @@
</TableLayout>
</androidx.core.widget.NestedScrollView>
<Button
android:id="@+id/select_demo_frameprocessors_button"
android:id="@+id/select_demo_effects_button"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="32dp"
android:layout_marginStart="32dp"
android:layout_marginEnd="32dp"
android:text="@string/select_demo_frameprocessors"
android:text="@string/select_demo_effects"
app:layout_constraintBottom_toTopOf="@+id/transform_button"
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintStart_toStartOf="parent" />
......
demos/transformer/src/main/res/values/strings.xml
......@@ -29,7 +29,7 @@
<string name="enable_fallback" translatable="false">Enable fallback</string>
<string name="request_sdr_tone_mapping" translatable="false">Request SDR tone-mapping (API 31+)</string>
<string name="hdr_editing" translatable="false">[Experimental] HDR editing</string>
<string name="select_demo_frameprocessors" translatable="false">Add demo effects</string>
<string name="select_demo_effects" translatable="false">Add demo effects</string>
<string name="periodic_vignette_options" translatable="false">Periodic vignette options</string>
<string name="transform" translatable="false">Transform</string>
<string name="debug_preview" translatable="false">Debug preview:</string>
......
libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java
......@@ -130,7 +130,7 @@ public final class FrameProcessorChainPixelTest {
Matrix translateRightMatrix = new Matrix();
translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0);
GlFrameProcessor glFrameProcessor = new AdvancedFrameProcessor(translateRightMatrix);
setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, glFrameProcessor);
setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, () -> glFrameProcessor);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_PNG_ASSET_PATH);
Bitmap actualBitmap = processFirstFrameAndEnd();
......@@ -155,7 +155,9 @@ public final class FrameProcessorChainPixelTest {
GlFrameProcessor rotate45FrameProcessor =
new ScaleToFitFrameProcessor.Builder().setRotationDegrees(45).build();
setUpAndPrepareFirstFrame(
DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, translateRightFrameProcessor, rotate45FrameProcessor);
DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO,
() -> translateRightFrameProcessor,
() -> rotate45FrameProcessor);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_THEN_ROTATE_PNG_ASSET_PATH);
Bitmap actualBitmap = processFirstFrameAndEnd();
......@@ -180,7 +182,9 @@ public final class FrameProcessorChainPixelTest {
GlFrameProcessor translateRightFrameProcessor =
new AdvancedFrameProcessor(translateRightMatrix);
setUpAndPrepareFirstFrame(
DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, rotate45FrameProcessor, translateRightFrameProcessor);
DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO,
() -> rotate45FrameProcessor,
() -> translateRightFrameProcessor);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_THEN_TRANSLATE_PNG_ASSET_PATH);
Bitmap actualBitmap = processFirstFrameAndEnd();
......@@ -200,7 +204,7 @@ public final class FrameProcessorChainPixelTest {
String testId = "processData_withPresentationFrameProcessor_setResolution";
GlFrameProcessor glFrameProcessor =
new PresentationFrameProcessor.Builder().setResolution(480).build();
setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, glFrameProcessor);
setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, () -> glFrameProcessor);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(REQUEST_OUTPUT_HEIGHT_PNG_ASSET_PATH);
Bitmap actualBitmap = processFirstFrameAndEnd();
......@@ -220,7 +224,7 @@ public final class FrameProcessorChainPixelTest {
String testId = "processData_withScaleToFitFrameProcessor_rotate45";
GlFrameProcessor glFrameProcessor =
new ScaleToFitFrameProcessor.Builder().setRotationDegrees(45).build();
setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, glFrameProcessor);
setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, () -> glFrameProcessor);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE45_SCALE_TO_FIT_PNG_ASSET_PATH);
Bitmap actualBitmap = processFirstFrameAndEnd();
......@@ -240,11 +244,10 @@ public final class FrameProcessorChainPixelTest {
* accessed on the {@link FrameProcessorChain}'s output {@code outputImageReader}.
*
* @param pixelWidthHeightRatio The ratio of width over height for each pixel.
* @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors} that will apply changes
* to the input frame.
* @param effects The {@link GlEffect GlEffects} to apply to the input frame.
*/
private void setUpAndPrepareFirstFrame(
float pixelWidthHeightRatio, GlFrameProcessor... frameProcessors) throws Exception {
private void setUpAndPrepareFirstFrame(float pixelWidthHeightRatio, GlEffect... effects)
throws Exception {
// Set up the extractor to read the first video frame and get its format.
MediaExtractor mediaExtractor = new MediaExtractor();
@Nullable MediaCodec mediaCodec = null;
......@@ -267,7 +270,7 @@ public final class FrameProcessorChainPixelTest {
pixelWidthHeightRatio,
inputWidth,
inputHeight,
asList(frameProcessors),
asList(effects),
/* enableExperimentalHdrEditing= */ false);
Size outputSize = frameProcessorChain.getOutputSize();
outputImageReader =
......
libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java
......@@ -112,16 +112,16 @@ public final class FrameProcessorChainTest {
private static FrameProcessorChain createFrameProcessorChainWithFakeFrameProcessors(
float pixelWidthHeightRatio, Size inputSize, List<Size> frameProcessorOutputSizes)
throws TransformationException {
ImmutableList.Builder<GlFrameProcessor> frameProcessors = new ImmutableList.Builder<>();
ImmutableList.Builder<GlEffect> effects = new ImmutableList.Builder<>();
for (Size element : frameProcessorOutputSizes) {
frameProcessors.add(new FakeFrameProcessor(element));
effects.add(() -> new FakeFrameProcessor(element));
}
return FrameProcessorChain.create(
getApplicationContext(),
pixelWidthHeightRatio,
inputSize.getWidth(),
inputSize.getHeight(),
frameProcessors.build(),
effects.build(),
/* enableExperimentalHdrEditing= */ false);
}
......
libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java
......@@ -75,7 +75,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
* by this ratio so that the output frame's pixels have a ratio of 1.
* @param inputWidth The input frame width, in pixels.
* @param inputHeight The input frame height, in pixels.
* @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors} to apply to each frame.
* @param effects The {@link GlEffect GlEffects} to apply to each frame.
* @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal.
* @return A new instance.
* @throws TransformationException If reading shader files fails, or an OpenGL error occurs while
......@@ -86,7 +86,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
float pixelWidthHeightRatio,
int inputWidth,
int inputHeight,
List<GlFrameProcessor> frameProcessors,
List<GlEffect> effects,
boolean enableExperimentalHdrEditing)
throws TransformationException {
checkArgument(inputWidth > 0, "inputWidth must be positive");
......@@ -103,7 +103,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
pixelWidthHeightRatio,
inputWidth,
inputHeight,
frameProcessors,
effects,
enableExperimentalHdrEditing,
singleThreadExecutorService))
.get();
......@@ -118,8 +118,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
}
/**
* Creates the OpenGL textures, framebuffers, initializes the {@link GlFrameProcessor
* GlFrameProcessors} and returns a new {@code FrameProcessorChain}.
* Creates the OpenGL textures and framebuffers, initializes the {@link GlFrameProcessor
* GlFrameProcessors} corresponding to the {@link GlEffect GlEffects}, and returns a new {@code
* FrameProcessorChain}.
*
* <p>This method must be executed using the {@code singleThreadExecutorService}.
*/
......@@ -129,15 +130,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
float pixelWidthHeightRatio,
int inputWidth,
int inputHeight,
List<GlFrameProcessor> frameProcessors,
List<GlEffect> effects,
boolean enableExperimentalHdrEditing,
ExecutorService singleThreadExecutorService)
throws IOException {
checkState(Thread.currentThread().getName().equals(THREAD_NAME));
EGLDisplay eglDisplay = GlUtil.createEglDisplay();
ExternalCopyFrameProcessor externalCopyFrameProcessor =
new ExternalCopyFrameProcessor(enableExperimentalHdrEditing);
EGLContext eglContext =
enableExperimentalHdrEditing
? GlUtil.createEglContextEs3Rgba1010102(eglDisplay)
......@@ -153,20 +152,21 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
GlUtil.focusPlaceholderEglSurface(eglContext, eglDisplay);
}
ImmutableList<GlFrameProcessor> expandedFrameProcessors =
getExpandedFrameProcessors(
externalCopyFrameProcessor, pixelWidthHeightRatio, frameProcessors);
ExternalCopyFrameProcessor externalCopyFrameProcessor =
new ExternalCopyFrameProcessor(enableExperimentalHdrEditing);
ImmutableList<GlFrameProcessor> frameProcessors =
getFrameProcessors(externalCopyFrameProcessor, pixelWidthHeightRatio, effects);
// Initialize frame processors.
int inputExternalTexId = GlUtil.createExternalTexture();
externalCopyFrameProcessor.initialize(context, inputExternalTexId, inputWidth, inputHeight);
int[] framebuffers = new int[expandedFrameProcessors.size() - 1];
int[] framebuffers = new int[frameProcessors.size() - 1];
Size inputSize = externalCopyFrameProcessor.getOutputSize();
for (int i = 1; i < expandedFrameProcessors.size(); i++) {
for (int i = 1; i < frameProcessors.size(); i++) {
int inputTexId = GlUtil.createTexture(inputSize.getWidth(), inputSize.getHeight());
framebuffers[i - 1] = GlUtil.createFboForTexture(inputTexId);
GlFrameProcessor frameProcessor = expandedFrameProcessors.get(i);
GlFrameProcessor frameProcessor = frameProcessors.get(i);
frameProcessor.initialize(context, inputTexId, inputSize.getWidth(), inputSize.getHeight());
inputSize = frameProcessor.getOutputSize();
}
......@@ -176,31 +176,34 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
singleThreadExecutorService,
inputExternalTexId,
framebuffers,
expandedFrameProcessors,
frameProcessors,
enableExperimentalHdrEditing);
}
private static ImmutableList<GlFrameProcessor> getExpandedFrameProcessors(
private static ImmutableList<GlFrameProcessor> getFrameProcessors(
ExternalCopyFrameProcessor externalCopyFrameProcessor,
float pixelWidthHeightRatio,
List<GlFrameProcessor> frameProcessors) {
ImmutableList.Builder<GlFrameProcessor> frameProcessorListBuilder =
List<GlEffect> effects) {
ImmutableList.Builder<GlFrameProcessor> frameProcessors =
new ImmutableList.Builder<GlFrameProcessor>().add(externalCopyFrameProcessor);
// Scale to expand the frame to apply the pixelWidthHeightRatio.
if (pixelWidthHeightRatio > 1f) {
frameProcessorListBuilder.add(
frameProcessors.add(
new ScaleToFitFrameProcessor.Builder()
.setScale(/* scaleX= */ pixelWidthHeightRatio, /* scaleY= */ 1f)
.build());
} else if (pixelWidthHeightRatio < 1f) {
frameProcessorListBuilder.add(
frameProcessors.add(
new ScaleToFitFrameProcessor.Builder()
.setScale(/* scaleX= */ 1f, /* scaleY= */ 1f / pixelWidthHeightRatio)
.build());
}
frameProcessorListBuilder.addAll(frameProcessors);
return frameProcessorListBuilder.build();
for (int i = 0; i < effects.size(); i++) {
frameProcessors.add(effects.get(i).toGlFrameProcessor());
}
return frameProcessors.build();
}
private static final String TAG = "FrameProcessorChain";
......
libraries/transformer/src/main/java/androidx/media3/transformer/GlEffect.java 0 → 100644
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer;
import androidx.media3.common.util.UnstableApi;
/**
* Interface for a video frame effect with a {@link GlFrameProcessor} implementation.
*
* <p>Implementations contain information specifying the effect and can be {@linkplain
* #toGlFrameProcessor() converted} to a {@link GlFrameProcessor} which applies the effect.
*/
@UnstableApi
public interface GlEffect {
/** Returns a {@link GlFrameProcessor} that applies the the effect. */
GlFrameProcessor toGlFrameProcessor();
}
libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
......@@ -104,7 +104,7 @@ public final class Transformer {
private boolean removeVideo;
private String containerMimeType;
private TransformationRequest transformationRequest;
private ImmutableList<GlFrameProcessor> frameProcessors;
private ImmutableList<GlEffect> videoFrameEffects;
private ListenerSet<Transformer.Listener> listeners;
private DebugViewProvider debugViewProvider;
private Looper looper;
......@@ -124,7 +124,7 @@ public final class Transformer {
debugViewProvider = DebugViewProvider.NONE;
containerMimeType = MimeTypes.VIDEO_MP4;
transformationRequest = new TransformationRequest.Builder().build();
frameProcessors = ImmutableList.of();
videoFrameEffects = ImmutableList.of();
}
/**
......@@ -142,7 +142,7 @@ public final class Transformer {
debugViewProvider = DebugViewProvider.NONE;
containerMimeType = MimeTypes.VIDEO_MP4;
transformationRequest = new TransformationRequest.Builder().build();
frameProcessors = ImmutableList.of();
videoFrameEffects = ImmutableList.of();
}
/** Creates a builder with the values of the provided {@link Transformer}. */
......@@ -154,7 +154,7 @@ public final class Transformer {
this.removeVideo = transformer.removeVideo;
this.containerMimeType = transformer.containerMimeType;
this.transformationRequest = transformer.transformationRequest;
this.frameProcessors = transformer.frameProcessors;
this.videoFrameEffects = transformer.videoFrameEffects;
this.listeners = transformer.listeners;
this.looper = transformer.looper;
this.encoderFactory = transformer.encoderFactory;
......@@ -187,20 +187,20 @@ public final class Transformer {
}
/**
* Sets the {@linkplain GlFrameProcessor frame processors} to apply to each frame.
* Sets the {@linkplain GlEffect effects} to apply to each video frame.
*
* <p>The {@linkplain GlFrameProcessor frame processors} are applied before any {@linkplain
* <p>The {@linkplain GlEffect effects} are applied before any {@linkplain
* TransformationRequest.Builder#setScale(float, float) scale}, {@linkplain
* TransformationRequest.Builder#setRotationDegrees(float) rotation}, or {@linkplain
* TransformationRequest.Builder#setResolution(int) resolution} changes specified in the {@link
* #setTransformationRequest(TransformationRequest) TransformationRequest} but after {@linkplain
* TransformationRequest.Builder#setFlattenForSlowMotion(boolean) slow-motion flattening}.
*
* @param frameProcessors The {@linkplain GlFrameProcessor frame processors}.
* @param effects The {@linkplain GlEffect effects} to apply to each video frame.
* @return This builder.
*/
public Builder setFrameProcessors(List<GlFrameProcessor> frameProcessors) {
this.frameProcessors = ImmutableList.copyOf(frameProcessors);
public Builder setVideoFrameEffects(List<GlEffect> effects) {
this.videoFrameEffects = ImmutableList.copyOf(effects);
return this;
}
......@@ -432,7 +432,7 @@ public final class Transformer {
removeVideo,
containerMimeType,
transformationRequest,
frameProcessors,
videoFrameEffects,
listeners,
looper,
clock,
......@@ -554,7 +554,7 @@ public final class Transformer {
private final boolean removeVideo;
private final String containerMimeType;
private final TransformationRequest transformationRequest;
private final ImmutableList<GlFrameProcessor> frameProcessors;
private final ImmutableList<GlEffect> videoFrameEffects;
private final Looper looper;
private final Clock clock;
private final Codec.EncoderFactory encoderFactory;
......@@ -575,7 +575,7 @@ public final class Transformer {
boolean removeVideo,
String containerMimeType,
TransformationRequest transformationRequest,
ImmutableList<GlFrameProcessor> frameProcessors,
ImmutableList<GlEffect> videoFrameEffects,
ListenerSet<Transformer.Listener> listeners,
Looper looper,
Clock clock,
......@@ -590,7 +590,7 @@ public final class Transformer {
this.removeVideo = removeVideo;
this.containerMimeType = containerMimeType;
this.transformationRequest = transformationRequest;
this.frameProcessors = frameProcessors;
this.videoFrameEffects = videoFrameEffects;
this.listeners = listeners;
this.looper = looper;
this.clock = clock;
......@@ -730,7 +730,7 @@ public final class Transformer {
removeAudio,
removeVideo,
transformationRequest,
frameProcessors,
videoFrameEffects,
encoderFactory,
decoderFactory,
new FallbackListener(mediaItem, listeners, transformationRequest),
......@@ -842,7 +842,7 @@ public final class Transformer {
private final boolean removeAudio;
private final boolean removeVideo;
private final TransformationRequest transformationRequest;
private final ImmutableList<GlFrameProcessor> frameProcessors;
private final ImmutableList<GlEffect> videoFrameEffects;
private final Codec.EncoderFactory encoderFactory;
private final Codec.DecoderFactory decoderFactory;
private final FallbackListener fallbackListener;
......@@ -854,7 +854,7 @@ public final class Transformer {
boolean removeAudio,
boolean removeVideo,
TransformationRequest transformationRequest,
ImmutableList<GlFrameProcessor> frameProcessors,
ImmutableList<GlEffect> videoFrameEffects,
Codec.EncoderFactory encoderFactory,
Codec.DecoderFactory decoderFactory,
FallbackListener fallbackListener,
......@@ -864,7 +864,7 @@ public final class Transformer {
this.removeAudio = removeAudio;
this.removeVideo = removeVideo;
this.transformationRequest = transformationRequest;
this.frameProcessors = frameProcessors;
this.videoFrameEffects = videoFrameEffects;
this.encoderFactory = encoderFactory;
this.decoderFactory = decoderFactory;
this.fallbackListener = fallbackListener;
......@@ -900,7 +900,7 @@ public final class Transformer {
muxerWrapper,
mediaClock,
transformationRequest,
frameProcessors,
videoFrameEffects,
encoderFactory,
decoderFactory,
fallbackListener,
......
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java
......@@ -35,7 +35,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private static final String TAG = "TVideoRenderer";
private final Context context;
private final ImmutableList<GlFrameProcessor> frameProcessors;
private final ImmutableList<GlEffect> effects;
private final Codec.EncoderFactory encoderFactory;
private final Codec.DecoderFactory decoderFactory;
private final Transformer.DebugViewProvider debugViewProvider;
......@@ -48,14 +48,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
MuxerWrapper muxerWrapper,
TransformerMediaClock mediaClock,
TransformationRequest transformationRequest,
ImmutableList<GlFrameProcessor> frameProcessors,
ImmutableList<GlEffect> effects,
Codec.EncoderFactory encoderFactory,
Codec.DecoderFactory decoderFactory,
FallbackListener fallbackListener,
Transformer.DebugViewProvider debugViewProvider) {
super(C.TRACK_TYPE_VIDEO, muxerWrapper, mediaClock, transformationRequest, fallbackListener);
this.context = context;
this.frameProcessors = frameProcessors;
this.effects = effects;
this.encoderFactory = encoderFactory;
this.decoderFactory = decoderFactory;
this.debugViewProvider = debugViewProvider;
......@@ -90,7 +90,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
context,
inputFormat,
transformationRequest,
frameProcessors,
effects,
decoderFactory,
encoderFactory,
muxerWrapper.getSupportedSampleMimeTypes(getTrackType()),
......@@ -137,7 +137,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
&& transformationRequest.outputHeight != inputFormat.height) {
return false;
}
if (!frameProcessors.isEmpty()) {
if (!effects.isEmpty()) {
return false;
}
return true;
......
libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
......@@ -50,7 +50,7 @@ import org.checkerframework.dataflow.qual.Pure;
Context context,
Format inputFormat,
TransformationRequest transformationRequest,
ImmutableList<GlFrameProcessor> frameProcessors,
ImmutableList<GlEffect> effects,
Codec.DecoderFactory decoderFactory,
Codec.EncoderFactory encoderFactory,
List<String> allowedOutputMimeTypes,
......@@ -68,33 +68,35 @@ import org.checkerframework.dataflow.qual.Pure;
int decodedHeight =
(inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width;
ImmutableList.Builder<GlFrameProcessor> frameProcessorsListBuilder =
new ImmutableList.Builder<GlFrameProcessor>().addAll(frameProcessors);
ImmutableList.Builder<GlEffect> effectsListBuilder =
new ImmutableList.Builder<GlEffect>().addAll(effects);
if (transformationRequest.scaleX != 1f
|| transformationRequest.scaleY != 1f
|| transformationRequest.rotationDegrees != 0f) {
frameProcessorsListBuilder.add(
new ScaleToFitFrameProcessor.Builder()
.setScale(transformationRequest.scaleX, transformationRequest.scaleY)
.setRotationDegrees(transformationRequest.rotationDegrees)
.build());
effectsListBuilder.add(
() ->
new ScaleToFitFrameProcessor.Builder()
.setScale(transformationRequest.scaleX, transformationRequest.scaleY)
.setRotationDegrees(transformationRequest.rotationDegrees)
.build());
}
if (transformationRequest.outputHeight != C.LENGTH_UNSET) {
frameProcessorsListBuilder.add(
new PresentationFrameProcessor.Builder()
.setResolution(transformationRequest.outputHeight)
.build());
effectsListBuilder.add(
() ->
new PresentationFrameProcessor.Builder()
.setResolution(transformationRequest.outputHeight)
.build());
}
EncoderCompatibilityFrameProcessor encoderCompatibilityFrameProcessor =
new EncoderCompatibilityFrameProcessor();
frameProcessorsListBuilder.add(encoderCompatibilityFrameProcessor);
effectsListBuilder.add(() -> encoderCompatibilityFrameProcessor);
frameProcessorChain =
FrameProcessorChain.create(
context,
inputFormat.pixelWidthHeightRatio,
/* inputWidth= */ decodedWidth,
/* inputHeight= */ decodedHeight,
frameProcessorsListBuilder.build(),
effectsListBuilder.build(),
transformationRequest.enableHdrEditing);
Size requestedEncoderSize = frameProcessorChain.getOutputSize();
outputRotationDegrees = encoderCompatibilityFrameProcessor.getOutputRotationDegrees();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment