Add playback tests for TS and MP4 samples that run on Robolectric

This commit introduces the infrastructure classes, and a couple of
illustrative usages.

PiperOrigin-RevId: 328301593

library/core/src/test/java/com/google/android/exoplayer2/e2etest/Mp4PlaybackTest.java

+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.android.exoplayer2.e2etest;
+
+import android.graphics.SurfaceTexture;
+import android.view.Surface;
+import androidx.test.core.app.ApplicationProvider;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.android.exoplayer2.MediaItem;
+import com.google.android.exoplayer2.Player;
+import com.google.android.exoplayer2.SimpleExoPlayer;
+import com.google.android.exoplayer2.testutil.AutoAdvancingFakeClock;
+import com.google.android.exoplayer2.testutil.DumpFileAsserts;
+import com.google.android.exoplayer2.testutil.PlaybackOutput;
+import com.google.android.exoplayer2.testutil.ShadowMediaCodecConfig;
+import com.google.android.exoplayer2.testutil.TestExoPlayer;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.robolectric.annotation.Config;
+
+/** End-to-end tests using MP4 samples. */
+// TODO(b/143232359): Remove once https://issuetracker.google.com/143232359 is resolved.
+@Config(sdk = 29)
+@RunWith(AndroidJUnit4.class)
+public class Mp4PlaybackTest {
+  @Rule
+  public ShadowMediaCodecConfig mediaCodecConfig =
+      ShadowMediaCodecConfig.forAllSupportedMimeTypes();
+
+  @Test
+  public void h264VideoAacAudio() throws Exception {
+    SimpleExoPlayer player =
+        new SimpleExoPlayer.Builder(ApplicationProvider.getApplicationContext())
+            .setClock(new AutoAdvancingFakeClock())
+            .build();
+    player.setVideoSurface(new Surface(new SurfaceTexture(/* texName= */ 1)));
+    PlaybackOutput playbackOutput = PlaybackOutput.register(player, mediaCodecConfig);
+
+    player.setMediaItem(MediaItem.fromUri("asset:///media/mp4/sample.mp4"));
+    player.prepare();
+    player.play();
+    TestExoPlayer.runUntilPlaybackState(player, Player.STATE_ENDED);
+
+    DumpFileAsserts.assertOutput(
+        ApplicationProvider.getApplicationContext(),
+        playbackOutput,
+        "playbackdumps/mp4/sample.mp4.dump");
+  }
+}

library/core/src/test/java/com/google/android/exoplayer2/e2etest/TsPlaybackTest.java

+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.e2etest;
+
+import android.graphics.SurfaceTexture;
+import android.view.Surface;
+import androidx.test.core.app.ApplicationProvider;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.android.exoplayer2.MediaItem;
+import com.google.android.exoplayer2.Player;
+import com.google.android.exoplayer2.SimpleExoPlayer;
+import com.google.android.exoplayer2.testutil.AutoAdvancingFakeClock;
+import com.google.android.exoplayer2.testutil.DumpFileAsserts;
+import com.google.android.exoplayer2.testutil.PlaybackOutput;
+import com.google.android.exoplayer2.testutil.ShadowMediaCodecConfig;
+import com.google.android.exoplayer2.testutil.TestExoPlayer;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.robolectric.annotation.Config;
+
+/** End-to-end tests using TS samples. */
+// TODO(b/143232359): Remove once https://issuetracker.google.com/143232359 is resolved.
+@Config(sdk = 29)
+@RunWith(AndroidJUnit4.class)
+public class TsPlaybackTest {
+
+  @Rule
+  public ShadowMediaCodecConfig mediaCodecConfig =
+      ShadowMediaCodecConfig.forAllSupportedMimeTypes();
+
+  @Test
+  public void mpegVideoMpegAudioScte35() throws Exception {
+    SimpleExoPlayer player =
+        new SimpleExoPlayer.Builder(ApplicationProvider.getApplicationContext())
+            .setClock(new AutoAdvancingFakeClock())
+            .build();
+    player.setVideoSurface(new Surface(new SurfaceTexture(/* texName= */ 1)));
+    PlaybackOutput playbackOutput = PlaybackOutput.register(player, mediaCodecConfig);
+
+    player.setMediaItem(MediaItem.fromUri("asset:///media/ts/sample_scte35.ts"));
+    player.prepare();
+    player.play();
+    TestExoPlayer.runUntilPlaybackState(player, Player.STATE_ENDED);
+
+    DumpFileAsserts.assertOutput(
+        ApplicationProvider.getApplicationContext(),
+        playbackOutput,
+        "playbackdumps/ts/sample_scte35.ts.dump");
+  }
+}

testdata/src/test/assets/playbackdumps/mp4/sample.mp4.dump

+MediaCodec (audio/mp4a-latm):
+  buffers.length = 46
+  buffers[0] = length 23, hash 47DE9131
+  buffers[1] = length 6, hash 31EC5206
+  buffers[2] = length 148, hash 894A176B
+  buffers[3] = length 189, hash CEF235A1
+  buffers[4] = length 205, hash BBF5F7B0
+  buffers[5] = length 210, hash F278B193
+  buffers[6] = length 210, hash 82DA1589
+  buffers[7] = length 207, hash 5BE231DF
+  buffers[8] = length 225, hash 18819EE1
+  buffers[9] = length 215, hash CA7FA67B
+  buffers[10] = length 211, hash 581A1C18
+  buffers[11] = length 216, hash ADB88187
+  buffers[12] = length 229, hash 2E8BA4DC
+  buffers[13] = length 232, hash 22F0C510
+  buffers[14] = length 235, hash 867AD0DC
+  buffers[15] = length 231, hash 84E823A8
+  buffers[16] = length 226, hash 1BEF3A95
+  buffers[17] = length 216, hash EAA345AE
+  buffers[18] = length 229, hash 6957411F
+  buffers[19] = length 219, hash 41275022
+  buffers[20] = length 241, hash 6495DF96
+  buffers[21] = length 228, hash 63D95906
+  buffers[22] = length 238, hash 34F676F9
+  buffers[23] = length 234, hash E5CBC045
+  buffers[24] = length 231, hash 5FC43661
+  buffers[25] = length 217, hash 682708ED
+  buffers[26] = length 239, hash D43780FC
+  buffers[27] = length 243, hash C5E17980
+  buffers[28] = length 231, hash AC5837BA
+  buffers[29] = length 230, hash 169EE895
+  buffers[30] = length 238, hash C48FF3F1
+  buffers[31] = length 225, hash 531E4599
+  buffers[32] = length 232, hash CB3C6B8D
+  buffers[33] = length 243, hash F8C94C7
+  buffers[34] = length 232, hash A646A7D0
+  buffers[35] = length 237, hash E8B787A5
+  buffers[36] = length 228, hash 3FA7A29F
+  buffers[37] = length 235, hash B9B33B0A
+  buffers[38] = length 264, hash 71A4869E
+  buffers[39] = length 257, hash D049B54C
+  buffers[40] = length 227, hash 66757231
+  buffers[41] = length 227, hash BD374F1B
+  buffers[42] = length 235, hash 999477F6
+  buffers[43] = length 229, hash FFF98DF0
+  buffers[44] = length 6, hash 31B22286
+  buffers[45] = length 0, hash 1
+MediaCodec (video/avc):
+  buffers.length = 31
+  buffers[0] = length 36692, hash D216076E
+  buffers[1] = length 5312, hash D45D3CA0
+  buffers[2] = length 599, hash 1BE7812D
+  buffers[3] = length 7735, hash 4490F110
+  buffers[4] = length 987, hash 560B5036
+  buffers[5] = length 673, hash ED7CD8C7
+  buffers[6] = length 523, hash 3020DF50
+  buffers[7] = length 6061, hash 736C72B2
+  buffers[8] = length 992, hash FE132F23
+  buffers[9] = length 623, hash 5B2C1816
+  buffers[10] = length 421, hash 742E69C1
+  buffers[11] = length 4899, hash F72F86A1
+  buffers[12] = length 568, hash 519A8E50
+  buffers[13] = length 620, hash 3990AA39
+  buffers[14] = length 5450, hash F06EC4AA
+  buffers[15] = length 1051, hash 92DFA63A
+  buffers[16] = length 874, hash 69587FB4
+  buffers[17] = length 781, hash 36BE495B
+  buffers[18] = length 4725, hash AC0C8CD3
+  buffers[19] = length 1022, hash 5D8BFF34
+  buffers[20] = length 790, hash 99413A99
+  buffers[21] = length 610, hash 5E129290
+  buffers[22] = length 2751, hash 769974CB
+  buffers[23] = length 745, hash B78A477A
+  buffers[24] = length 621, hash CF741E7A
+  buffers[25] = length 505, hash 1DB4894E
+  buffers[26] = length 1268, hash C15348DC
+  buffers[27] = length 880, hash C2DE85D0
+  buffers[28] = length 530, hash C98BC6A8
+  buffers[29] = length 568, hash 4FE5C8EA
+  buffers[30] = length 0, hash 1

testdata/src/test/assets/playbackdumps/ts/sample_scte35.ts.dump

+MediaCodec (audio/mpeg-L2):
+  buffers.length = 5
+  buffers[0] = length 1253, hash 727FD1C6
+  buffers[1] = length 1254, hash 73FB07B8
+  buffers[2] = length 1254, hash 73FB07B8
+  buffers[3] = length 1254, hash 73FB07B8
+  buffers[4] = length 0, hash 1
+MediaCodec (video/mpeg2):
+  buffers.length = 3
+  buffers[0] = length 20711, hash 34341E8
+  buffers[1] = length 18112, hash EC44B35B
+  buffers[2] = length 0, hash 1
+MetadataOutput:
+  Metadata[0]:
+    entry[0] = SpliceInsertCommand
+  Metadata[1]:
+    entry[0] = SpliceInsertCommand
+  Metadata[2]:
+    entry[0] = SpliceInsertCommand

testutils/build.gradle

@@ -24,6 +24,7 @@ dependencies {
     compileOnly 'org.jetbrains.kotlin:kotlin-annotations-jvm:' + kotlinAnnotationsVersion
     implementation 'androidx.annotation:annotation:' + androidxAnnotationVersion
     implementation project(modulePrefix + 'library-core')
+    implementation 'org.robolectric:robolectric:' + robolectricVersion
     testImplementation 'org.robolectric:robolectric:' + robolectricVersion
 }
 

testutils/src/main/java/com/google/android/exoplayer2/testutil/PlaybackOutput.java

+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.testutil;
+
+import com.google.android.exoplayer2.SimpleExoPlayer;
+import com.google.android.exoplayer2.metadata.Metadata;
+import com.google.android.exoplayer2.util.Assertions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Class to capture output from a playback test.
+ *
+ * <p>Implements {@link Dumper.Dumpable} so the output can be easily dumped to a string for
+ * comparison against previous test runs.
+ */
+public final class PlaybackOutput implements Dumper.Dumpable {
+
+  private final ShadowMediaCodecConfig codecConfig;
+
+  // TODO: Add support for subtitles too
+  private final List<Metadata> metadatas;
+
+  private PlaybackOutput(SimpleExoPlayer player, ShadowMediaCodecConfig codecConfig) {
+    this.codecConfig = codecConfig;
+
+    metadatas = Collections.synchronizedList(new ArrayList<>());
+    // TODO: Consider passing playback position into MetadataOutput and TextOutput. Calling
+    // player.getCurrentPosition() inside onMetadata/Cues will likely be non-deterministic
+    // because renderer-thread != playback-thread.
+    player.addMetadataOutput(metadatas::add);
+  }
+
+  /**
+   * Create an instance that captures the metadata and text output from {@code player} and the audio
+   * and video output via the {@link TeeCodec TeeCodecs} exposed by {@code mediaCodecConfig}.
+   *
+   * <p>Must be called <b>before</b> playback to ensure metadata and text output is captured
+   * correctly.
+   *
+   * @param player The {@link SimpleExoPlayer} to capture metadata and text output from.
+   * @param mediaCodecConfig The {@link ShadowMediaCodecConfig} to capture audio and video output
+   *     from.
+   * @return A new instance that can be used to dump the playback output.
+   */
+  public static PlaybackOutput register(
+      SimpleExoPlayer player, ShadowMediaCodecConfig mediaCodecConfig) {
+    return new PlaybackOutput(player, mediaCodecConfig);
+  }
+
+  @Override
+  public void dump(Dumper dumper) {
+    ImmutableMap<String, TeeCodec> codecs = codecConfig.getCodecs();
+    ImmutableList<String> mimeTypes = ImmutableList.sortedCopyOf(codecs.keySet());
+    for (String mimeType : mimeTypes) {
+      dumper.add(Assertions.checkNotNull(codecs.get(mimeType)));
+    }
+
+    dumpMetadata(dumper);
+  }
+
+  private void dumpMetadata(Dumper dumper) {
+    if (metadatas.isEmpty()) {
+      return;
+    }
+    dumper.startBlock("MetadataOutput");
+    for (int i = 0; i < metadatas.size(); i++) {
+      dumper.startBlock("Metadata[" + i + "]");
+      Metadata metadata = metadatas.get(i);
+      for (int j = 0; j < metadata.length(); j++) {
+        dumper.add("entry[" + j + "]", metadata.get(j).getClass().getSimpleName());
+      }
+      dumper.endBlock();
+    }
+    dumper.endBlock();
+  }
+}

testutils/src/main/java/com/google/android/exoplayer2/testutil/ShadowMediaCodecConfig.java

+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.testutil;
+
+import android.media.MediaCodecInfo;
+import android.media.MediaFormat;
+import com.google.android.exoplayer2.util.MimeTypes;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.primitives.Ints;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.junit.rules.ExternalResource;
+import org.robolectric.shadows.MediaCodecInfoBuilder;
+import org.robolectric.shadows.ShadowMediaCodec;
+import org.robolectric.shadows.ShadowMediaCodecList;
+
+/**
+ * A JUnit @Rule to configure Roboelectric's {@link ShadowMediaCodec}.
+ *
+ * <p>Registers a {@link org.robolectric.shadows.ShadowMediaCodec.CodecConfig} for each audio/video
+ * MIME type known by ExoPlayer, and provides access to the bytes passed to these via {@link
+ * TeeCodec}.
+ */
+public final class ShadowMediaCodecConfig extends ExternalResource {
+
+  private final Map<String, TeeCodec> codecsByMimeType;
+
+  private ShadowMediaCodecConfig() {
+    this.codecsByMimeType = new HashMap<>();
+  }
+
+  public static ShadowMediaCodecConfig forAllSupportedMimeTypes() {
+    return new ShadowMediaCodecConfig();
+  }
+
+  public ImmutableMap<String, TeeCodec> getCodecs() {
+    return ImmutableMap.copyOf(codecsByMimeType);
+  }
+
+  @Override
+  protected void before() throws Throwable {
+    // Video codecs
+    MediaCodecInfo.CodecProfileLevel avcProfileLevel =
+        createProfileLevel(
+            MediaCodecInfo.CodecProfileLevel.AVCProfileHigh,
+            MediaCodecInfo.CodecProfileLevel.AVCLevel62);
+    configureCodec(
+        /* codecName= */ "exotest.video.avc",
+        MimeTypes.VIDEO_H264,
+        ImmutableList.of(avcProfileLevel),
+        ImmutableList.of(MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible));
+    MediaCodecInfo.CodecProfileLevel mpeg2ProfileLevel =
+        createProfileLevel(
+            MediaCodecInfo.CodecProfileLevel.MPEG2ProfileMain,
+            MediaCodecInfo.CodecProfileLevel.MPEG2LevelML);
+    configureCodec(
+        /* codecName= */ "exotest.video.mpeg2",
+        MimeTypes.VIDEO_MPEG2,
+        ImmutableList.of(mpeg2ProfileLevel),
+        ImmutableList.of(MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible));
+
+    // Audio codecs
+    configureCodec("exotest.audio.aac", MimeTypes.AUDIO_AAC);
+    configureCodec("exotest.audio.mpegl2", MimeTypes.AUDIO_MPEG_L2);
+  }
+
+  @Override
+  protected void after() {
+    codecsByMimeType.clear();
+    ShadowMediaCodecList.reset();
+    ShadowMediaCodec.clearCodecs();
+  }
+
+  private void configureCodec(String codecName, String mimeType) {
+    configureCodec(
+        codecName,
+        mimeType,
+        /* profileLevels= */ ImmutableList.of(),
+        /* colorFormats= */ ImmutableList.of());
+  }
+
+  private void configureCodec(
+      String codecName,
+      String mimeType,
+      List<MediaCodecInfo.CodecProfileLevel> profileLevels,
+      List<Integer> colorFormats) {
+    MediaFormat mediaFormat = new MediaFormat();
+    mediaFormat.setString(MediaFormat.KEY_MIME, mimeType);
+    MediaCodecInfoBuilder.CodecCapabilitiesBuilder capabilities =
+        MediaCodecInfoBuilder.CodecCapabilitiesBuilder.newBuilder().setMediaFormat(mediaFormat);
+    if (!profileLevels.isEmpty()) {
+      capabilities.setProfileLevels(profileLevels.toArray(new MediaCodecInfo.CodecProfileLevel[0]));
+    }
+    if (!colorFormats.isEmpty()) {
+      capabilities.setColorFormats(Ints.toArray(colorFormats));
+    }
+    ShadowMediaCodecList.addCodec(
+        MediaCodecInfoBuilder.newBuilder()
+            .setName(codecName)
+            .setCapabilities(capabilities.build())
+            .build());
+    // TODO: Update ShadowMediaCodec to consider the MediaFormat.KEY_MAX_INPUT_SIZE value passed
+    // to configure() so we don't have to specify large buffers here.
+    TeeCodec codec = new TeeCodec(mimeType);
+    ShadowMediaCodec.addDecoder(
+        codecName,
+        new ShadowMediaCodec.CodecConfig(
+            /* inputBufferSize= */ 50_000, /* outputBufferSize= */ 50_000, codec));
+    codecsByMimeType.put(mimeType, codec);
+  }
+
+  private static MediaCodecInfo.CodecProfileLevel createProfileLevel(int profile, int level) {
+    MediaCodecInfo.CodecProfileLevel profileLevel = new MediaCodecInfo.CodecProfileLevel();
+    profileLevel.profile = profile;
+    profileLevel.level = level;
+    return profileLevel;
+  }
+}

testutils/src/main/java/com/google/android/exoplayer2/testutil/TeeCodec.java

+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.testutil;
+
+import com.google.android.exoplayer2.util.MimeTypes;
+import com.google.common.collect.ImmutableList;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.robolectric.shadows.ShadowMediaCodec;
+
+/**
+ * A {@link ShadowMediaCodec.CodecConfig.Codec} for Robolectric's {@link ShadowMediaCodec} that
+ * records the contents of buffers passed to it before copying the contents into the output buffer.
+ *
+ * <p>This also implements {@link Dumper.Dumpable} so the recorded buffers can be written out to a
+ * dump file.
+ */
+public final class TeeCodec implements ShadowMediaCodec.CodecConfig.Codec, Dumper.Dumpable {
+
+  private final String mimeType;
+  private final List<byte[]> receivedBuffers;
+
+  public TeeCodec(String mimeType) {
+    this.mimeType = mimeType;
+    this.receivedBuffers = Collections.synchronizedList(new ArrayList<>());
+  }
+
+  @Override
+  public void process(ByteBuffer in, ByteBuffer out) {
+    byte[] bytes = new byte[in.remaining()];
+    in.get(bytes);
+    receivedBuffers.add(bytes);
+
+    if (!MimeTypes.isAudio(mimeType)) {
+      // Don't output audio bytes, because ShadowAudioTrack doesn't advance the playback position so
+      // playback never completes.
+      // TODO: Update ShadowAudioTrack to advance the playback position in a realistic way.
+      out.put(bytes);
+    }
+  }
+
+  @Override
+  public void dump(Dumper dumper) {
+    if (receivedBuffers.isEmpty()) {
+      return;
+    }
+    dumper.startBlock("MediaCodec (" + mimeType + ")");
+    dumper.add("buffers.length", receivedBuffers.size());
+    for (int i = 0; i < receivedBuffers.size(); i++) {
+      dumper.add("buffers[" + i + "]", receivedBuffers.get(i));
+    }
+
+    dumper.endBlock();
+  }
+
+  /**
+   * Return the buffers received by this codec.
+   *
+   * <p>The list is sorted in the order the buffers were passed to {@link #process(ByteBuffer,
+   * ByteBuffer)}.
+   */
+  public ImmutableList<byte[]> getReceivedBuffers() {
+    return ImmutableList.copyOf(receivedBuffers);
+  }
+}