Create InAppMuxer in transformer

To use the InAppMuxer, the client needs to pass InAppMuxer Factory.

PiperOrigin-RevId: 530684007

library/muxer/src/main/java/androidx/media3/muxer/Boxes.java

@@ -562,7 +562,7 @@ import java.util.Locale;
     contents.put(paspBox());
 
     // Put in a "colr" box if any of the three color format parameters has a non-default (0) value.
-    // TODO(b/278101856): Only null check should be enough once we disallow invalid values.
+    // TODO: b/278101856 - Only null check should be enough once we disallow invalid values.
     if (format.colorInfo != null
         && (format.colorInfo.colorSpace != 0
             || format.colorInfo.colorTransfer != 0
@@ -588,6 +588,7 @@ import java.util.Locale;
    * @param lastDurationBehavior The behaviour for the last sample duration.
    * @return A list of all the sample durations.
    */
+  // TODO: b/280084657 - Add support for setting last sample duration.
   public static List<Long> durationsVuForStts(
       List<MediaCodec.BufferInfo> writtenSamples,
       long minInputPresentationTimestampUs,
@@ -671,7 +672,7 @@ import java.util.Locale;
 
     contents.putInt(0x0); // version and flags.
 
-    // TODO(b/270583563): Consider optimizing for identically-sized samples.
+    // TODO: b/270583563 - Consider optimizing for identically-sized samples.
     //  sample_size; specifying the default sample size. Set to zero to indicate that the samples
     //  have different sizes and they are stored in the sample size table.
     contents.putInt(0);
@@ -697,7 +698,7 @@ import java.util.Locale;
 
     int currentChunk = 1;
 
-    // TODO(b/270583563): Consider optimizing for consecutive chunks having same number of samples.
+    // TODO: b/270583563 - Consider optimizing for consecutive chunks having same number of samples.
     for (int i = 0; i < writtenChunkSampleCounts.size(); i++) {
       int samplesInChunk = writtenChunkSampleCounts.get(i);
       contents.putInt(currentChunk); // first_chunk.

library/muxer/src/main/java/androidx/media3/muxer/Mp4Muxer.java

@@ -23,6 +23,8 @@ import androidx.annotation.FloatRange;
 import androidx.annotation.IntDef;
 import androidx.annotation.Nullable;
 import com.google.android.exoplayer2.Format;
+import com.google.android.exoplayer2.util.MimeTypes;
+import com.google.common.collect.ImmutableList;
 import com.google.errorprone.annotations.CanIgnoreReturnValue;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -138,6 +140,14 @@ public final class Mp4Muxer {
     }
   }
 
+  /** A list of supported video sample mime types. */
+  public static final ImmutableList<String> SUPPORTED_VIDEO_SAMPLE_MIME_TYPES =
+      ImmutableList.of(MimeTypes.VIDEO_H264, MimeTypes.VIDEO_H265, MimeTypes.VIDEO_AV1);
+
+  /** A list of supported audio sample mime types. */
+  public static final ImmutableList<String> SUPPORTED_AUDIO_SAMPLE_MIME_TYPES =
+      ImmutableList.of(MimeTypes.AUDIO_AAC);
+
   private final Mp4Writer mp4Writer;
   private final MetadataCollector metadataCollector;
 

library/muxer/src/main/java/androidx/media3/muxer/Mp4Writer.java

@@ -319,7 +319,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
   private void doInterleave() throws IOException {
     for (int i = 0; i < tracks.size(); i++) {
       Track track = tracks.get(i);
-      // TODO(b/270583563): check if we need to consider the global timestamp instead.
+      // TODO: b/270583563 - check if we need to consider the global timestamp instead.
       if (track.pendingSamples.size() > 2) {
         BufferInfo firstSampleInfo = checkNotNull(track.pendingSamples.peekFirst()).first;
         BufferInfo lastSampleInfo = checkNotNull(track.pendingSamples.peekLast()).first;
@@ -377,9 +377,21 @@ import java.util.concurrent.atomic.AtomicBoolean;
       }
 
       // Skip empty samples.
-      // TODO(b/279931840): Confirm whether muxer should throw when writing empty samples.
+      // TODO: b/279931840 - Confirm whether muxer should throw when writing empty samples.
       if (byteBuffer.remaining() > 0) {
-        pendingSamples.addLast(Pair.create(bufferInfo, byteBuffer));
+        // Copy sample data and release the original buffer.
+        ByteBuffer byteBufferCopy = ByteBuffer.allocateDirect(byteBuffer.remaining());
+        byteBufferCopy.put(byteBuffer);
+        byteBufferCopy.rewind();
+
+        BufferInfo bufferInfoCopy = new BufferInfo();
+        bufferInfoCopy.set(
+            /* newOffset= */ byteBufferCopy.position(),
+            /* newSize= */ byteBufferCopy.remaining(),
+            bufferInfo.presentationTimeUs,
+            bufferInfo.flags);
+
+        pendingSamples.addLast(Pair.create(bufferInfoCopy, byteBufferCopy));
         doInterleave();
       }
     }
@@ -387,7 +399,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
     @Override
     public int videoUnitTimebase() {
       return MimeTypes.isAudio(format.sampleMimeType)
-          ? 48_000 // TODO(b/270583563): Update these with actual values from mediaFormat.
+          ? 48_000 // TODO: b/270583563 - Update these with actual values from mediaFormat.
           : 90_000;
     }
 

library/transformer/build.gradle

@@ -39,6 +39,7 @@ dependencies {
     implementation project(modulePrefix + 'library-datasource')
     implementation project(modulePrefix + 'library-core')
     implementation project(modulePrefix + 'library-effect')
+    implementation project(modulePrefix + 'library-muxer')
     compileOnly 'com.google.errorprone:error_prone_annotations:' + errorProneVersion
     compileOnly 'org.checkerframework:checker-qual:' + checkerframeworkVersion
     compileOnly 'org.checkerframework:checker-compat-qual:' + checkerframeworkCompatVersion

library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/AndroidTestUtil.java

@@ -17,6 +17,7 @@ package com.google.android.exoplayer2.transformer;
 
 import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
 import static com.google.android.exoplayer2.util.Assertions.checkState;
+import static com.google.android.exoplayer2.util.MimeTypes.VIDEO_AV1;
 import static com.google.android.exoplayer2.util.MimeTypes.VIDEO_H264;
 import static com.google.android.exoplayer2.util.MimeTypes.VIDEO_H265;
 
@@ -66,6 +67,15 @@ public final class AndroidTestUtil {
           .setCodecs("avc1.64001F")
           .build();
 
+  public static final String MP4_ASSET_AV1_VIDEO_URI_STRING = "asset:///media/mp4/sample_av1.mp4";
+  public static final Format MP4_ASSET_AV1_VIDEO_FORMAT =
+      new Format.Builder()
+          .setSampleMimeType(VIDEO_AV1)
+          .setWidth(1080)
+          .setHeight(720)
+          .setFrameRate(30.0f)
+          .build();
+
   public static final String MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING =
       "asset:///media/mp4/sample_with_increasing_timestamps.mp4";
   public static final Format MP4_ASSET_WITH_INCREASING_TIMESTAMPS_FORMAT =

library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerWithInAppMuxerEndToEndTest.java

+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.transformer;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import android.content.Context;
+import android.net.Uri;
+import androidx.test.core.app.ApplicationProvider;
+import com.google.android.exoplayer2.MediaItem;
+import com.google.android.exoplayer2.audio.ChannelMixingAudioProcessor;
+import com.google.android.exoplayer2.audio.ChannelMixingMatrix;
+import com.google.android.exoplayer2.effect.RgbFilter;
+import com.google.android.exoplayer2.util.Effect;
+import com.google.common.collect.ImmutableList;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+/** End-to-end instrumentation test for {@link Transformer} with {@link InAppMuxer}. */
+@RunWith(Parameterized.class)
+public class TransformerWithInAppMuxerEndToEndTest {
+  private static final String MP4_FILE_ASSET_DIRECTORY = "asset:///media/mp4/";
+  private static final String H264_MP4 = "sample.mp4";
+  private static final String H265_MP4 = "h265_with_metadata_track.mp4";
+
+  @Parameters(name = "{0}")
+  public static ImmutableList<String> mediaFiles() {
+    return ImmutableList.of(H264_MP4, H265_MP4);
+  }
+
+  @Parameter public @MonotonicNonNull String inputFile;
+
+  private final Context context = ApplicationProvider.getApplicationContext();
+
+  @Test
+  public void videoEditing_completesSuccessfully() throws Exception {
+    String testId = "videoEditing_completesSuccessfully";
+    Transformer transformer =
+        new Transformer.Builder(context)
+            .setMuxerFactory(
+                new InAppMuxer.Factory(DefaultMuxer.Factory.DEFAULT_MAX_DELAY_BETWEEN_SAMPLES_MS))
+            .build();
+    ImmutableList<Effect> videoEffects = ImmutableList.of(RgbFilter.createGrayscaleFilter());
+    MediaItem mediaItem = MediaItem.fromUri(Uri.parse(MP4_FILE_ASSET_DIRECTORY + inputFile));
+    EditedMediaItem editedMediaItem =
+        new EditedMediaItem.Builder(mediaItem)
+            .setEffects(new Effects(/* audioProcessors= */ ImmutableList.of(), videoEffects))
+            .build();
+
+    ExportTestResult result =
+        new TransformerAndroidTestRunner.Builder(context, transformer)
+            .build()
+            .run(testId, editedMediaItem);
+
+    assertThat(result.exportResult.exportException).isNull();
+  }
+
+  @Test
+  public void audioEditing_completesSuccessfully() throws Exception {
+    String testId = "audioEditing_completesSuccessfully";
+    Transformer transformer =
+        new Transformer.Builder(context)
+            .setMuxerFactory(
+                new InAppMuxer.Factory(DefaultMuxer.Factory.DEFAULT_MAX_DELAY_BETWEEN_SAMPLES_MS))
+            .build();
+    ChannelMixingAudioProcessor channelMixingAudioProcessor = new ChannelMixingAudioProcessor();
+    channelMixingAudioProcessor.putChannelMixingMatrix(
+        ChannelMixingMatrix.create(/* inputChannelCount= */ 1, /* outputChannelCount= */ 2));
+    MediaItem mediaItem = MediaItem.fromUri(Uri.parse(MP4_FILE_ASSET_DIRECTORY + H264_MP4));
+    EditedMediaItem editedMediaItem =
+        new EditedMediaItem.Builder(mediaItem)
+            .setEffects(
+                new Effects(
+                    ImmutableList.of(channelMixingAudioProcessor),
+                    /* videoEffects= */ ImmutableList.of()))
+            .build();
+
+    ExportTestResult result =
+        new TransformerAndroidTestRunner.Builder(context, transformer)
+            .build()
+            .run(testId, editedMediaItem);
+
+    assertThat(result.exportResult.exportException).isNull();
+  }
+}

library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/TransformerWithInAppMuxerEndToEndTest.java

+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.transformer.mh;
+
+import static com.google.android.exoplayer2.transformer.AndroidTestUtil.MP4_ASSET_AV1_VIDEO_FORMAT;
+import static com.google.android.exoplayer2.transformer.AndroidTestUtil.MP4_ASSET_AV1_VIDEO_URI_STRING;
+import static com.google.common.truth.Truth.assertThat;
+
+import android.content.Context;
+import android.net.Uri;
+import androidx.test.core.app.ApplicationProvider;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.android.exoplayer2.MediaItem;
+import com.google.android.exoplayer2.effect.RgbFilter;
+import com.google.android.exoplayer2.transformer.AndroidTestUtil;
+import com.google.android.exoplayer2.transformer.DefaultMuxer;
+import com.google.android.exoplayer2.transformer.EditedMediaItem;
+import com.google.android.exoplayer2.transformer.Effects;
+import com.google.android.exoplayer2.transformer.ExportTestResult;
+import com.google.android.exoplayer2.transformer.InAppMuxer;
+import com.google.android.exoplayer2.transformer.Transformer;
+import com.google.android.exoplayer2.transformer.TransformerAndroidTestRunner;
+import com.google.android.exoplayer2.util.Effect;
+import com.google.common.collect.ImmutableList;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/** End-to-end instrumentation test for {@link Transformer} with {@link InAppMuxer}. */
+@RunWith(AndroidJUnit4.class)
+public class TransformerWithInAppMuxerEndToEndTest {
+  @Test
+  public void videoEditing_forAv1Video_completesSuccessfully() throws Exception {
+    String testId = "videoEditing_forAv1Video_completesSuccessfully";
+    Context context = ApplicationProvider.getApplicationContext();
+    if (AndroidTestUtil.skipAndLogIfFormatsUnsupported(
+        context, testId, /* inputFormat= */ MP4_ASSET_AV1_VIDEO_FORMAT, /* outputFormat= */ null)) {
+      return;
+    }
+    Transformer transformer =
+        new Transformer.Builder(context)
+            .setMuxerFactory(
+                new InAppMuxer.Factory(DefaultMuxer.Factory.DEFAULT_MAX_DELAY_BETWEEN_SAMPLES_MS))
+            .build();
+    ImmutableList<Effect> videoEffects = ImmutableList.of(RgbFilter.createGrayscaleFilter());
+    MediaItem mediaItem = MediaItem.fromUri(Uri.parse(MP4_ASSET_AV1_VIDEO_URI_STRING));
+    EditedMediaItem editedMediaItem =
+        new EditedMediaItem.Builder(mediaItem)
+            .setEffects(new Effects(/* audioProcessors= */ ImmutableList.of(), videoEffects))
+            .build();
+
+    ExportTestResult result =
+        new TransformerAndroidTestRunner.Builder(context, transformer)
+            .build()
+            .run(testId, editedMediaItem);
+
+    assertThat(result.exportResult.exportException).isNull();
+  }
+}

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameworkMuxer.java

@@ -162,7 +162,7 @@ import java.nio.ByteBuffer;
     int offset = data.position();
     int size = data.limit() - offset;
 
-    bufferInfo.set(offset, size, presentationTimeUs, getMediaMuxerFlags(flags));
+    bufferInfo.set(offset, size, presentationTimeUs, TransformerUtil.getMediaCodecFlags(flags));
     long lastSamplePresentationTimeUs = trackIndexToLastPresentationTimeUs.get(trackIndex);
     // writeSampleData blocks on old API versions, so check here to avoid calling the method.
     checkState(
@@ -231,17 +231,6 @@ import java.nio.ByteBuffer;
     return maxDelayBetweenSamplesMs;
   }
 
-  private static int getMediaMuxerFlags(@C.BufferFlags int flags) {
-    int mediaMuxerFlags = 0;
-    if ((flags & C.BUFFER_FLAG_KEY_FRAME) == C.BUFFER_FLAG_KEY_FRAME) {
-      mediaMuxerFlags |= MediaCodec.BUFFER_FLAG_KEY_FRAME;
-    }
-    if ((flags & C.BUFFER_FLAG_END_OF_STREAM) == C.BUFFER_FLAG_END_OF_STREAM) {
-      mediaMuxerFlags |= MediaCodec.BUFFER_FLAG_END_OF_STREAM;
-    }
-    return mediaMuxerFlags;
-  }
-
   // Accesses MediaMuxer state via reflection to ensure that muxer resources can be released even
   // if stopping fails.
   @SuppressLint("PrivateApi")

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/InAppMuxer.java

+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.transformer;
+
+import static androidx.media3.muxer.Mp4Muxer.SUPPORTED_AUDIO_SAMPLE_MIME_TYPES;
+import static androidx.media3.muxer.Mp4Muxer.SUPPORTED_VIDEO_SAMPLE_MIME_TYPES;
+
+import android.media.MediaCodec.BufferInfo;
+import androidx.media3.muxer.Mp4Muxer;
+import androidx.media3.muxer.Mp4Muxer.TrackToken;
+import com.google.android.exoplayer2.C;
+import com.google.android.exoplayer2.Format;
+import com.google.android.exoplayer2.metadata.Metadata;
+import com.google.android.exoplayer2.metadata.mp4.Mp4LocationData;
+import com.google.android.exoplayer2.util.MimeTypes;
+import com.google.common.collect.ImmutableList;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+/** {@link Muxer} implementation that uses a {@link Mp4Muxer}. */
+public final class InAppMuxer implements Muxer {
+  /** {@link Muxer.Factory} for {@link InAppMuxer}. */
+  public static final class Factory implements Muxer.Factory {
+    private final long maxDelayBetweenSamplesMs;
+
+    /** {@link Muxer.Factory} for {@link InAppMuxer}. */
+    public Factory(long maxDelayBetweenSamplesMs) {
+      this.maxDelayBetweenSamplesMs = maxDelayBetweenSamplesMs;
+    }
+
+    @Override
+    public InAppMuxer create(String path) throws MuxerException {
+      FileOutputStream outputStream;
+      try {
+        outputStream = new FileOutputStream(path);
+      } catch (FileNotFoundException e) {
+        throw new MuxerException("Error creating file output stream", e);
+      }
+
+      Mp4Muxer mp4Muxer = new Mp4Muxer.Builder(outputStream).build();
+      return new InAppMuxer(mp4Muxer, maxDelayBetweenSamplesMs);
+    }
+
+    @Override
+    public ImmutableList<String> getSupportedSampleMimeTypes(@C.TrackType int trackType) {
+      if (trackType == C.TRACK_TYPE_VIDEO) {
+        return SUPPORTED_VIDEO_SAMPLE_MIME_TYPES;
+      } else if (trackType == C.TRACK_TYPE_AUDIO) {
+        return SUPPORTED_AUDIO_SAMPLE_MIME_TYPES;
+      }
+      return ImmutableList.of();
+    }
+  }
+
+  private final Mp4Muxer mp4Muxer;
+  private final long maxDelayBetweenSamplesMs;
+  private final List<TrackToken> trackTokenList;
+  private final BufferInfo bufferInfo;
+
+  private InAppMuxer(Mp4Muxer mp4Muxer, long maxDelayBetweenSamplesMs) {
+    this.mp4Muxer = mp4Muxer;
+    this.maxDelayBetweenSamplesMs = maxDelayBetweenSamplesMs;
+    trackTokenList = new ArrayList<>();
+    bufferInfo = new BufferInfo();
+  }
+
+  @Override
+  public int addTrack(Format format) {
+    // Keep same sort key as no specific sort order is required.
+    TrackToken trackToken = mp4Muxer.addTrack(/* sortKey= */ 0, format);
+    trackTokenList.add(trackToken);
+
+    if (MimeTypes.isVideo(format.sampleMimeType)) {
+      mp4Muxer.setOrientation(format.rotationDegrees);
+    }
+
+    return trackTokenList.size() - 1;
+  }
+
+  @Override
+  public void writeSampleData(
+      int trackIndex, ByteBuffer data, long presentationTimeUs, @C.BufferFlags int flags)
+      throws MuxerException {
+
+    int size = data.remaining();
+    bufferInfo.set(
+        data.position(), size, presentationTimeUs, TransformerUtil.getMediaCodecFlags(flags));
+
+    try {
+      mp4Muxer.writeSampleData(trackTokenList.get(trackIndex), data, bufferInfo);
+    } catch (IOException e) {
+      throw new MuxerException(
+          "Failed to write sample for trackIndex="
+              + trackIndex
+              + ", presentationTimeUs="
+              + presentationTimeUs
+              + ", size="
+              + size,
+          e);
+    }
+  }
+
+  @Override
+  public void addMetadata(Metadata metadata) {
+    for (int i = 0; i < metadata.length(); i++) {
+      Metadata.Entry entry = metadata.get(i);
+      if (entry instanceof Mp4LocationData) {
+        mp4Muxer.setLocation(
+            ((Mp4LocationData) entry).latitude, ((Mp4LocationData) entry).longitude);
+      }
+    }
+  }
+
+  @Override
+  public void release(boolean forCancellation) throws MuxerException {
+    try {
+      mp4Muxer.close();
+    } catch (IOException e) {
+      throw new MuxerException("Error closing muxer", e);
+    }
+  }
+
+  @Override
+  public long getMaxDelayBetweenSamplesMs() {
+    return maxDelayBetweenSamplesMs;
+  }
+}

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerUtil.java

@@ -16,6 +16,7 @@
 
 package com.google.android.exoplayer2.transformer;
 
+import android.media.MediaCodec;
 import androidx.annotation.Nullable;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.Format;
@@ -84,4 +85,16 @@ import com.google.common.collect.ImmutableList;
     }
     return false;
   }
+
+  /** Returns {@link MediaCodec} flags corresponding to {@link C.BufferFlags}. */
+  public static int getMediaCodecFlags(@C.BufferFlags int flags) {
+    int mediaCodecFlags = 0;
+    if ((flags & C.BUFFER_FLAG_KEY_FRAME) == C.BUFFER_FLAG_KEY_FRAME) {
+      mediaCodecFlags |= MediaCodec.BUFFER_FLAG_KEY_FRAME;
+    }
+    if ((flags & C.BUFFER_FLAG_END_OF_STREAM) == C.BUFFER_FLAG_END_OF_STREAM) {
+      mediaCodecFlags |= MediaCodec.BUFFER_FLAG_END_OF_STREAM;
+    }
+    return mediaCodecFlags;
+  }
 }