Add test for audio and video from different sources

PiperOrigin-RevId: 515379858

library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerAndroidTestRunner.java

@@ -15,6 +15,7 @@
  */
 package com.google.android.exoplayer2.transformer;
 
+import static com.google.android.exoplayer2.util.Assertions.checkArgument;
 import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
 import static java.util.concurrent.TimeUnit.SECONDS;
 
@@ -31,6 +32,7 @@ import com.google.android.exoplayer2.util.Log;
 import com.google.android.exoplayer2.util.SystemClock;
 import com.google.android.exoplayer2.util.Util;
 import com.google.common.base.Ascii;
+import com.google.common.collect.ImmutableList;
 import com.google.errorprone.annotations.CanIgnoreReturnValue;
 import java.io.File;
 import java.io.IOException;
@@ -174,20 +176,20 @@ public class TransformerAndroidTestRunner {
   }
 
   /**
-   * Exports the {@link EditedMediaItem}, saving a summary of the export to the application cache.
+   * Exports the {@link Composition}, saving a summary of the export to the application cache.
    *
    * @param testId A unique identifier for the transformer test run.
-   * @param editedMediaItem The {@link EditedMediaItem} to export.
+   * @param composition The {@link Composition} to export.
    * @return The {@link ExportTestResult}.
    * @throws Exception The cause of the export not completing.
    */
-  public ExportTestResult run(String testId, EditedMediaItem editedMediaItem) throws Exception {
+  public ExportTestResult run(String testId, Composition composition) throws Exception {
     JSONObject resultJson = new JSONObject();
     if (inputValues != null) {
       resultJson.put("inputValues", JSONObject.wrap(inputValues));
     }
     try {
-      ExportTestResult exportTestResult = runInternal(testId, editedMediaItem);
+      ExportTestResult exportTestResult = runInternal(testId, composition);
       resultJson.put("exportResult", exportTestResult.asJsonObject());
       if (exportTestResult.exportResult.exportException != null) {
         throw exportTestResult.exportResult.exportException;
@@ -210,6 +212,21 @@ public class TransformerAndroidTestRunner {
   }
 
   /**
+   * Exports the {@link EditedMediaItem}, saving a summary of the export to the application cache.
+   *
+   * @param testId A unique identifier for the transformer test run.
+   * @param editedMediaItem The {@link EditedMediaItem} to export.
+   * @return The {@link ExportTestResult}.
+   * @throws Exception The cause of the export not completing.
+   */
+  public ExportTestResult run(String testId, EditedMediaItem editedMediaItem) throws Exception {
+    EditedMediaItemSequence sequence =
+        new EditedMediaItemSequence(ImmutableList.of(editedMediaItem));
+    Composition composition = new Composition.Builder(ImmutableList.of(sequence)).build();
+    return run(testId, composition);
+  }
+
+  /**
    * Exports the {@link MediaItem}, saving a summary of the export to the application cache.
    *
    * @param testId A unique identifier for the transformer test run.
@@ -223,33 +240,49 @@ public class TransformerAndroidTestRunner {
   }
 
   /**
-   * Exports the {@link EditedMediaItem}.
+   * Exports the {@link Composition}.
    *
    * @param testId An identifier for the test.
-   * @param editedMediaItem The {@link EditedMediaItem} to export.
+   * @param composition The {@link Composition} to export.
    * @return The {@link ExportTestResult}.
-   * @throws IllegalStateException See {@link Transformer#start(EditedMediaItem, String)}.
+   * @throws IllegalStateException See {@link Transformer#start(Composition, String)}.
    * @throws InterruptedException If the thread is interrupted whilst waiting for transformer to
    *     complete.
    * @throws IOException If an error occurs opening the output file for writing.
    * @throws TimeoutException If the export has not completed after {@linkplain
    *     Builder#setTimeoutSeconds(int) the given timeout}.
    */
-  private ExportTestResult runInternal(String testId, EditedMediaItem editedMediaItem)
+  private ExportTestResult runInternal(String testId, Composition composition)
       throws InterruptedException, IOException, TimeoutException {
-    MediaItem mediaItem = editedMediaItem.mediaItem;
-    if (!mediaItem.clippingConfiguration.equals(MediaItem.ClippingConfiguration.UNSET)
-        && requestCalculateSsim) {
-      throw new UnsupportedOperationException(
+    if (requestCalculateSsim) {
+      checkArgument(
+          composition.sequences.size() == 1
+              && composition.sequences.get(0).editedMediaItems.size() == 1,
+          "SSIM is only relevant for single MediaItem compositions");
+      checkArgument(
+          composition
+              .sequences
+              .get(0)
+              .editedMediaItems
+              .get(0)
+              .mediaItem
+              .clippingConfiguration
+              .equals(MediaItem.ClippingConfiguration.UNSET),
           "SSIM calculation is not supported for clipped inputs.");
     }
-
-    Uri mediaItemUri = checkNotNull(mediaItem.localConfiguration).uri;
-    String scheme = checkNotNull(mediaItemUri.getScheme());
-    if ((scheme.equals("http") || scheme.equals("https")) && !hasNetworkConnection(context)) {
-      throw new UnsupportedOperationException(
-          "Input network file requested on device with no network connection. Input file name: "
-              + mediaItemUri);
+    if (!hasNetworkConnection(context)) {
+      for (EditedMediaItemSequence sequence : composition.sequences) {
+        for (EditedMediaItem editedMediaItem : sequence.editedMediaItems) {
+          Uri mediaItemUri = checkNotNull(editedMediaItem.mediaItem.localConfiguration).uri;
+          String scheme = checkNotNull(mediaItemUri.getScheme());
+          if ((scheme.equals("http") || scheme.equals("https"))) {
+            throw new IllegalArgumentException(
+                "Input network file requested on device with no network connection. Input file"
+                    + " name: "
+                    + mediaItemUri);
+          }
+        }
+      }
     }
 
     AtomicReference<@NullableType FallbackDetails> fallbackDetailsReference =
@@ -307,7 +340,7 @@ public class TransformerAndroidTestRunner {
         .runOnMainSync(
             () -> {
               try {
-                testTransformer.start(editedMediaItem, outputVideoFile.getAbsolutePath());
+                testTransformer.start(composition, outputVideoFile.getAbsolutePath());
                 // Catch all exceptions to report. Exceptions thrown here and not caught will NOT
                 // propagate.
               } catch (Exception e) {
@@ -359,6 +392,7 @@ public class TransformerAndroidTestRunner {
       return testResultBuilder.build();
     }
     try {
+      MediaItem mediaItem = composition.sequences.get(0).editedMediaItems.get(0).mediaItem;
       double ssim =
           SsimHelper.calculate(
               context,

library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java

@@ -28,7 +28,10 @@ import androidx.test.ext.junit.runners.AndroidJUnit4;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.MediaItem;
+import com.google.android.exoplayer2.audio.AudioProcessor;
+import com.google.android.exoplayer2.audio.SonicAudioProcessor;
 import com.google.android.exoplayer2.effect.Presentation;
+import com.google.android.exoplayer2.effect.RgbFilter;
 import com.google.android.exoplayer2.util.Effect;
 import com.google.common.collect.ImmutableList;
 import org.junit.Test;
@@ -185,6 +188,51 @@ public class TransformerEndToEndTest {
     assertThat(exception).hasMessageThat().contains("video");
   }
 
+  @Test
+  public void start_audioVideoTranscodedFromDifferentSequences_producesExpectedResult()
+      throws Exception {
+    Transformer transformer = new Transformer.Builder(context).build();
+    String testId = "start_audioVideoTranscodedFromDifferentSequences_producesExpectedResult";
+    ImmutableList<AudioProcessor> audioProcessors = ImmutableList.of(new SonicAudioProcessor());
+    ImmutableList<Effect> videoEffects = ImmutableList.of(RgbFilter.createGrayscaleFilter());
+    MediaItem mediaItem = MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING));
+    EditedMediaItem editedMediaItem =
+        new EditedMediaItem.Builder(mediaItem)
+            .setEffects(new Effects(audioProcessors, videoEffects))
+            .build();
+    ExportTestResult expectedResult =
+        new TransformerAndroidTestRunner.Builder(context, transformer)
+            .build()
+            .run(testId, editedMediaItem);
+    EditedMediaItem audioEditedMediaItem =
+        new EditedMediaItem.Builder(mediaItem)
+            .setEffects(new Effects(audioProcessors, /* videoEffects= */ ImmutableList.of()))
+            .setRemoveVideo(true)
+            .build();
+    EditedMediaItemSequence audioSequence =
+        new EditedMediaItemSequence(ImmutableList.of(audioEditedMediaItem));
+    EditedMediaItem videoEditedMediaItem =
+        new EditedMediaItem.Builder(mediaItem)
+            .setEffects(new Effects(/* audioProcessors= */ ImmutableList.of(), videoEffects))
+            .setRemoveAudio(true)
+            .build();
+    EditedMediaItemSequence videoSequence =
+        new EditedMediaItemSequence(ImmutableList.of(videoEditedMediaItem));
+    Composition composition =
+        new Composition.Builder(ImmutableList.of(audioSequence, videoSequence)).build();
+
+    ExportTestResult result =
+        new TransformerAndroidTestRunner.Builder(context, transformer)
+            .build()
+            .run(testId, composition);
+
+    assertThat(result.exportResult.channelCount)
+        .isEqualTo(expectedResult.exportResult.channelCount);
+    assertThat(result.exportResult.videoFrameCount)
+        .isEqualTo(expectedResult.exportResult.videoFrameCount);
+    assertThat(result.exportResult.durationMs).isEqualTo(expectedResult.exportResult.durationMs);
+  }
+
   private static final class VideoUnsupportedEncoderFactory implements Codec.EncoderFactory {
 
     private final Codec.EncoderFactory encoderFactory;

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoAssetLoaderBaseRenderer.java

@@ -22,7 +22,6 @@ import static com.google.android.exoplayer2.transformer.AssetLoader.SUPPORTED_OU
 import static com.google.android.exoplayer2.transformer.AssetLoader.SUPPORTED_OUTPUT_TYPE_ENCODED;
 import static com.google.android.exoplayer2.transformer.TransformerUtil.getProcessedTrackType;
 import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
-import static com.google.android.exoplayer2.util.Assertions.checkState;
 
 import androidx.annotation.Nullable;
 import com.google.android.exoplayer2.BaseRenderer;
@@ -54,6 +53,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
 
   private boolean isRunning;
   private long streamStartPositionUs;
+  private boolean shouldInitDecoder;
 
   public ExoAssetLoaderBaseRenderer(
       @C.TrackType int trackType,
@@ -97,7 +97,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
   @Override
   public void render(long positionUs, long elapsedRealtimeUs) {
     try {
-      if (!isRunning || isEnded() || !hasReadInputFormat()) {
+      if (!isRunning || isEnded() || !readInputFormatAndInitDecoderIfNeeded()) {
         return;
       }
 
@@ -195,35 +195,39 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
    *     {@linkplain Codec decoder}.
    */
   @EnsuresNonNullIf(expression = "inputFormat", result = true)
-  private boolean hasReadInputFormat() throws ExportException {
-    if (inputFormat != null) {
+  private boolean readInputFormatAndInitDecoderIfNeeded() throws ExportException {
+    if (inputFormat != null && !shouldInitDecoder) {
       return true;
     }
 
-    FormatHolder formatHolder = getFormatHolder();
-    @ReadDataResult
-    int result = readSource(formatHolder, decoderInputBuffer, /* readFlags= */ FLAG_REQUIRE_FORMAT);
-    if (result != C.RESULT_FORMAT_READ) {
-      return false;
+    if (inputFormat == null) {
+      FormatHolder formatHolder = getFormatHolder();
+      @ReadDataResult
+      int result =
+          readSource(formatHolder, decoderInputBuffer, /* readFlags= */ FLAG_REQUIRE_FORMAT);
+      if (result != C.RESULT_FORMAT_READ) {
+        return false;
+      }
+      inputFormat = overrideFormat(checkNotNull(formatHolder.format));
+      onInputFormatRead(inputFormat);
+      shouldInitDecoder =
+          assetLoaderListener.onTrackAdded(
+              inputFormat,
+              SUPPORTED_OUTPUT_TYPE_DECODED | SUPPORTED_OUTPUT_TYPE_ENCODED,
+              streamStartPositionUs,
+              streamOffsetUs);
     }
-    inputFormat = overrideFormat(checkNotNull(formatHolder.format));
-    onInputFormatRead(inputFormat);
-
-    boolean decodeOutput =
-        assetLoaderListener.onTrackAdded(
-            inputFormat,
-            SUPPORTED_OUTPUT_TYPE_DECODED | SUPPORTED_OUTPUT_TYPE_ENCODED,
-            streamStartPositionUs,
-            streamOffsetUs);
-    if (decodeOutput) {
-      if (getProcessedTrackType(inputFormat.sampleMimeType) == C.TRACK_TYPE_AUDIO) {
-        initDecoder(inputFormat);
-      } else {
+
+    if (shouldInitDecoder) {
+      if (getProcessedTrackType(inputFormat.sampleMimeType) == C.TRACK_TYPE_VIDEO) {
         // TODO(b/237674316): Move surface creation out of video sampleConsumer. Init decoder and
         // get decoder output Format before init sampleConsumer.
-        checkState(ensureSampleConsumerInitialized());
-        initDecoder(inputFormat);
+        if (!ensureSampleConsumerInitialized()) {
+          return false;
+        }
       }
+      initDecoder(inputFormat);
+      shouldInitDecoder = false;
     }
 
     return true;

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java

@@ -700,7 +700,6 @@ public final class Transformer {
    * @throws IllegalStateException If an export is already in progress.
    */
   public void start(Composition composition, String path) {
-    checkArgument(composition.sequences.size() == 1);
     checkArgument(composition.effects.audioProcessors.isEmpty());
     // Only supports Presentation in video effects.
     ImmutableList<Effect> videoEffects = composition.effects.videoEffects;

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerInternal.java

@@ -465,7 +465,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       } else {
         outputHasVideo.set(true);
       }
-      if (trackCountsToReport.get() == 0 && tracksToAdd.decrementAndGet() == 0) {
+      if (tracksToAdd.decrementAndGet() == 0 && trackCountsToReport.get() == 0) {
         int outputTrackCount = (outputHasAudio.get() ? 1 : 0) + (outputHasVideo.get() ? 1 : 0);
         muxerWrapper.setTrackCount(outputTrackCount);
         fallbackListener.setTrackCount(outputTrackCount);

library/transformer/src/test/java/com/google/android/exoplayer2/transformer/CompositionExportTest.java

+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.transformer;
+
+import static com.google.android.exoplayer2.transformer.TestUtil.ASSET_URI_PREFIX;
+import static com.google.android.exoplayer2.transformer.TestUtil.FILE_AUDIO_VIDEO;
+import static com.google.android.exoplayer2.transformer.TestUtil.createTransformerBuilder;
+import static com.google.common.truth.Truth.assertThat;
+
+import android.content.Context;
+import androidx.test.core.app.ApplicationProvider;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.android.exoplayer2.MediaItem;
+import com.google.android.exoplayer2.transformer.TestUtil.TestMuxerFactory.TestMuxerHolder;
+import com.google.android.exoplayer2.util.Util;
+import com.google.common.collect.ImmutableList;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/**
+ * End-to-end test for exporting a {@link Composition} containing multiple {@link
+ * EditedMediaItemSequence} instances with {@link Transformer}.
+ */
+@RunWith(AndroidJUnit4.class)
+public class CompositionExportTest {
+
+  private String outputPath;
+  private TestMuxerHolder testMuxerHolder;
+
+  @Before
+  public void setUp() throws Exception {
+    Context context = ApplicationProvider.getApplicationContext();
+    outputPath = Util.createTempFile(context, "TransformerTest").getPath();
+    testMuxerHolder = new TestUtil.TestMuxerFactory.TestMuxerHolder();
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    Files.delete(Paths.get(outputPath));
+  }
+
+  @Test
+  public void start_audioVideoTransmuxedFromDifferentSequences_producesExpectedResult()
+      throws Exception {
+    Transformer transformer =
+        createTransformerBuilder(testMuxerHolder, /* enableFallback= */ false).build();
+    MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_VIDEO);
+    transformer.start(mediaItem, outputPath);
+    ExportResult expectedExportResult = TransformerTestRunner.runLooper(transformer);
+    EditedMediaItem audioEditedMediaItem =
+        new EditedMediaItem.Builder(mediaItem).setRemoveVideo(true).build();
+    EditedMediaItemSequence audioSequence =
+        new EditedMediaItemSequence(ImmutableList.of(audioEditedMediaItem));
+    EditedMediaItem videoEditedMediaItem =
+        new EditedMediaItem.Builder(mediaItem).setRemoveAudio(true).build();
+    EditedMediaItemSequence videoSequence =
+        new EditedMediaItemSequence(ImmutableList.of(videoEditedMediaItem));
+    Composition composition =
+        new Composition.Builder(ImmutableList.of(audioSequence, videoSequence))
+            .setTransmuxAudio(true)
+            .setTransmuxVideo(true)
+            .build();
+
+    transformer.start(composition, outputPath);
+    ExportResult exportResult = TransformerTestRunner.runLooper(transformer);
+
+    // We can't compare the muxer output against a dump file because the asset loaders in each
+    // sequence load samples from their own thread, independently of each other, which makes the
+    // output non-deterministic.
+    assertThat(exportResult.channelCount).isEqualTo(expectedExportResult.channelCount);
+    assertThat(exportResult.videoFrameCount).isEqualTo(expectedExportResult.videoFrameCount);
+    assertThat(exportResult.durationMs).isEqualTo(expectedExportResult.durationMs);
+  }
+}