Skip to content

SDK / exoplayer

Go to a project
Commit b8c8ce0e by ibaker Committed by Oliver Woodman
Options

Use Mp4WebvttDecoder for WebVTT content in DASH MP4 containers 

This was broken by https://github.com/google/ExoPlayer/commit/74a9d8f680995f2096c59fde6cd1ef6e85bb4d55
because DashManifestParser switched to setting Format.sampleMimeType to
text/vtt while SubtitleDecoderFactory was still expecting
application/x-mp4-vtt. This change teaches SubtitleDecoderFactory to
check both Format.containerMimeType and Format.sampleMimeType.

I'll investigate a follow-up change to remove
MimeTypes.APPLICATION_MP4VTT completely (it's currently still used in
AtomParsers).

Issue: #7985
PiperOrigin-RevId: 334771672
parent 9ce2ac8a
Inline Side-by-side
Showing with 229 additions and 7 deletions
RELEASENOTES.md
...@@ -11,6 +11,8 @@ ...@@ -11,6 +11,8 @@
([#7866](https://github.com/google/ExoPlayer/issues/7866)). ([#7866](https://github.com/google/ExoPlayer/issues/7866)).
* Text: * Text:
* Add support for `\h` SSA/ASS style override code (non-breaking space). * Add support for `\h` SSA/ASS style override code (non-breaking space).
* Fix WebVTT subtitles in MP4 containers in DASH streams
([#7985](https://github.com/google/ExoPlayer/issues/7985)).
* UI: * UI:
* Do not require subtitleButton in custom layouts of StyledPlayerView * Do not require subtitleButton in custom layouts of StyledPlayerView
([#7962](https://github.com/google/ExoPlayer/issues/7962)). ([#7962](https://github.com/google/ExoPlayer/issues/7962)).
......
library/core/src/main/java/com/google/android/exoplayer2/text/SubtitleDecoderFactory.java
...@@ -91,11 +91,15 @@ public interface SubtitleDecoderFactory { ...@@ -91,11 +91,15 @@ public interface SubtitleDecoderFactory {
@Override @Override
public SubtitleDecoder createDecoder(Format format) { public SubtitleDecoder createDecoder(Format format) {
@Nullable String mimeType = format.sampleMimeType; @Nullable String sampleMimeType = format.sampleMimeType;
if (mimeType != null) { if (sampleMimeType != null) {
switch (mimeType) { switch (sampleMimeType) {
case MimeTypes.TEXT_VTT: case MimeTypes.TEXT_VTT:
return new WebvttDecoder(); if (MimeTypes.APPLICATION_MP4.equals(format.containerMimeType)) {
return new Mp4WebvttDecoder();
} else {
return new WebvttDecoder();
}
case MimeTypes.TEXT_SSA: case MimeTypes.TEXT_SSA:
return new SsaDecoder(format.initializationData); return new SsaDecoder(format.initializationData);
case MimeTypes.APPLICATION_MP4VTT: case MimeTypes.APPLICATION_MP4VTT:
...@@ -109,7 +113,7 @@ public interface SubtitleDecoderFactory { ...@@ -109,7 +113,7 @@ public interface SubtitleDecoderFactory {
case MimeTypes.APPLICATION_CEA608: case MimeTypes.APPLICATION_CEA608:
case MimeTypes.APPLICATION_MP4CEA608: case MimeTypes.APPLICATION_MP4CEA608:
return new Cea608Decoder( return new Cea608Decoder(
mimeType, sampleMimeType,
format.accessibilityChannel, format.accessibilityChannel,
Cea608Decoder.MIN_DATA_CHANNEL_TIMEOUT_MS); Cea608Decoder.MIN_DATA_CHANNEL_TIMEOUT_MS);
case MimeTypes.APPLICATION_CEA708: case MimeTypes.APPLICATION_CEA708:
...@@ -123,7 +127,7 @@ public interface SubtitleDecoderFactory { ...@@ -123,7 +127,7 @@ public interface SubtitleDecoderFactory {
} }
} }
throw new IllegalArgumentException( throw new IllegalArgumentException(
"Attempted to create decoder for unsupported MIME type: " + mimeType); "Attempted to create decoder for unsupported MIME type: " + sampleMimeType);
} }
}; };
} }
library/dash/build.gradle
...@@ -36,6 +36,7 @@ dependencies { ...@@ -36,6 +36,7 @@ dependencies {
compileOnly 'org.checkerframework:checker-compat-qual:' + checkerframeworkCompatVersion compileOnly 'org.checkerframework:checker-compat-qual:' + checkerframeworkCompatVersion
compileOnly 'org.jetbrains.kotlin:kotlin-annotations-jvm:' + kotlinAnnotationsVersion compileOnly 'org.jetbrains.kotlin:kotlin-annotations-jvm:' + kotlinAnnotationsVersion
implementation 'androidx.annotation:annotation:' + androidxAnnotationVersion implementation 'androidx.annotation:annotation:' + androidxAnnotationVersion
testImplementation project(modulePrefix + 'robolectricutils')
testImplementation project(modulePrefix + 'testutils') testImplementation project(modulePrefix + 'testutils')
testImplementation 'org.robolectric:robolectric:' + robolectricVersion testImplementation 'org.robolectric:robolectric:' + robolectricVersion
} }
......
library/dash/src/test/java/com/google/android/exoplayer2/e2etest/DashPlaybackTest.java 0 → 100644
/*
* Copyright (C) 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.e2etest;
import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import android.graphics.SurfaceTexture;
import android.view.Surface;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.android.exoplayer2.MediaItem;
import com.google.android.exoplayer2.Player;
import com.google.android.exoplayer2.SimpleExoPlayer;
import com.google.android.exoplayer2.robolectric.PlaybackOutput;
import com.google.android.exoplayer2.robolectric.ShadowMediaCodecConfig;
import com.google.android.exoplayer2.testutil.AutoAdvancingFakeClock;
import com.google.android.exoplayer2.testutil.DumpFileAsserts;
import com.google.android.exoplayer2.testutil.TestExoPlayer;
import com.google.android.exoplayer2.trackselection.DefaultTrackSelector;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.robolectric.annotation.Config;
/** End-to-end tests using DASH samples. */
// TODO(b/143232359): Remove once https://issuetracker.google.com/143232359 is resolved.
@Config(sdk = 29)
@RunWith(AndroidJUnit4.class)
public final class DashPlaybackTest {
@Rule
public ShadowMediaCodecConfig mediaCodecConfig =
ShadowMediaCodecConfig.forAllSupportedMimeTypes();
// https://github.com/google/ExoPlayer/issues/7985
@Test
public void webvttInMp4() throws Exception {
SimpleExoPlayer player =
new SimpleExoPlayer.Builder(ApplicationProvider.getApplicationContext())
.setClock(new AutoAdvancingFakeClock())
.build();
player.setVideoSurface(new Surface(new SurfaceTexture(/* texName= */ 1)));
PlaybackOutput playbackOutput = PlaybackOutput.register(player, mediaCodecConfig);
// Ensure the subtitle track is selected.
DefaultTrackSelector trackSelector =
checkNotNull((DefaultTrackSelector) player.getTrackSelector());
trackSelector.setParameters(trackSelector.buildUponParameters().setPreferredTextLanguage("en"));
player.setMediaItem(MediaItem.fromUri("asset:///media/dash/webvtt-in-mp4/sample.mpd"));
player.prepare();
player.play();
TestExoPlayer.runUntilPlaybackState(player, Player.STATE_ENDED);
player.release();
DumpFileAsserts.assertOutput(
ApplicationProvider.getApplicationContext(),
playbackOutput,
"playbackdumps/dash/webvtt-in-mp4.dump");
}
}
robolectricutils/src/main/java/com/google/android/exoplayer2/robolectric/PlaybackOutput.java
...@@ -15,12 +15,17 @@ ...@@ -15,12 +15,17 @@
*/ */
package com.google.android.exoplayer2.robolectric; package com.google.android.exoplayer2.robolectric;
import android.graphics.Bitmap;
import androidx.annotation.Nullable;
import com.google.android.exoplayer2.SimpleExoPlayer; import com.google.android.exoplayer2.SimpleExoPlayer;
import com.google.android.exoplayer2.metadata.Metadata; import com.google.android.exoplayer2.metadata.Metadata;
import com.google.android.exoplayer2.testutil.Dumper; import com.google.android.exoplayer2.testutil.Dumper;
import com.google.android.exoplayer2.text.Cue;
import com.google.android.exoplayer2.util.Assertions; import com.google.android.exoplayer2.util.Assertions;
import com.google.android.exoplayer2.util.Util;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import java.nio.ByteBuffer;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
...@@ -35,17 +40,19 @@ public final class PlaybackOutput implements Dumper.Dumpable { ...@@ -35,17 +40,19 @@ public final class PlaybackOutput implements Dumper.Dumpable {
private final ShadowMediaCodecConfig codecConfig; private final ShadowMediaCodecConfig codecConfig;
// TODO: Add support for subtitles too
private final List<Metadata> metadatas; private final List<Metadata> metadatas;
private final List<List<Cue>> subtitles;
private PlaybackOutput(SimpleExoPlayer player, ShadowMediaCodecConfig codecConfig) { private PlaybackOutput(SimpleExoPlayer player, ShadowMediaCodecConfig codecConfig) {
this.codecConfig = codecConfig; this.codecConfig = codecConfig;
metadatas = Collections.synchronizedList(new ArrayList<>()); metadatas = Collections.synchronizedList(new ArrayList<>());
subtitles = Collections.synchronizedList(new ArrayList<>());
// TODO: Consider passing playback position into MetadataOutput and TextOutput. Calling // TODO: Consider passing playback position into MetadataOutput and TextOutput. Calling
// player.getCurrentPosition() inside onMetadata/Cues will likely be non-deterministic // player.getCurrentPosition() inside onMetadata/Cues will likely be non-deterministic
// because renderer-thread != playback-thread. // because renderer-thread != playback-thread.
player.addMetadataOutput(metadatas::add); player.addMetadataOutput(metadatas::add);
player.addTextOutput(subtitles::add);
} }
/** /**
...@@ -74,6 +81,7 @@ public final class PlaybackOutput implements Dumper.Dumpable { ...@@ -74,6 +81,7 @@ public final class PlaybackOutput implements Dumper.Dumpable {
} }
dumpMetadata(dumper); dumpMetadata(dumper);
dumpSubtitles(dumper);
} }
private void dumpMetadata(Dumper dumper) { private void dumpMetadata(Dumper dumper) {
...@@ -91,4 +99,57 @@ public final class PlaybackOutput implements Dumper.Dumpable { ...@@ -91,4 +99,57 @@ public final class PlaybackOutput implements Dumper.Dumpable {
} }
dumper.endBlock(); dumper.endBlock();
} }
private void dumpSubtitles(Dumper dumper) {
if (subtitles.isEmpty()) {
return;
}
dumper.startBlock("TextOutput");
for (int i = 0; i < subtitles.size(); i++) {
dumper.startBlock("Subtitle[" + i + "]");
List<Cue> subtitle = subtitles.get(i);
if (subtitle.isEmpty()) {
dumper.add("Cues", ImmutableList.of());
}
for (int j = 0; j < subtitle.size(); j++) {
dumper.startBlock("Cue[" + j + "]");
Cue cue = subtitle.get(j);
dumpIfNotEqual(dumper, "text", cue.text, null);
dumpIfNotEqual(dumper, "textAlignment", cue.textAlignment, null);
dumpBitmap(dumper, cue.bitmap);
dumpIfNotEqual(dumper, "line", cue.line, Cue.DIMEN_UNSET);
dumpIfNotEqual(dumper, "lineType", cue.lineType, Cue.TYPE_UNSET);
dumpIfNotEqual(dumper, "lineAnchor", cue.lineAnchor, Cue.TYPE_UNSET);
dumpIfNotEqual(dumper, "position", cue.position, Cue.DIMEN_UNSET);
dumpIfNotEqual(dumper, "positionAnchor", cue.positionAnchor, Cue.TYPE_UNSET);
dumpIfNotEqual(dumper, "size", cue.size, Cue.DIMEN_UNSET);
dumpIfNotEqual(dumper, "bitmapHeight", cue.bitmapHeight, Cue.DIMEN_UNSET);
if (cue.windowColorSet) {
dumper.add("cue.windowColor", cue.windowColor);
}
dumpIfNotEqual(dumper, "textSizeType", cue.textSizeType, Cue.TYPE_UNSET);
dumpIfNotEqual(dumper, "textSize", cue.textSize, Cue.DIMEN_UNSET);
dumpIfNotEqual(dumper, "verticalType", cue.verticalType, Cue.TYPE_UNSET);
dumper.endBlock();
}
dumper.endBlock();
}
dumper.endBlock();
}
private static void dumpIfNotEqual(
Dumper dumper, String field, @Nullable Object actual, @Nullable Object comparison) {
if (!Util.areEqual(actual, comparison)) {
dumper.add(field, actual);
}
}
private static void dumpBitmap(Dumper dumper, @Nullable Bitmap bitmap) {
if (bitmap == null) {
return;
}
byte[] bytes = new byte[bitmap.getByteCount()];
bitmap.copyPixelsToBuffer(ByteBuffer.wrap(bytes));
dumper.add("bitmap", bytes);
}
} }
testdata/src/test/assets/media/dash/webvtt-in-mp4/sample.mpd 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<!--Generated with https://github.com/google/shaka-packager version v2.4.1-c731217-release-->
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" minBufferTime="PT2S" type="static" mediaPresentationDuration="PT1.0010000467300415S">
<Period id="0">
<AdaptationSet id="0" contentType="text" subsegmentAlignment="true" lang="eng">
<Role schemeIdUri="urn:mpeg:dash:role:2011" value="subtitle"/>
<Representation id="0" bandwidth="314" codecs="wvtt" mimeType="application/mp4">
<BaseURL>sample.text.mp4</BaseURL>
<SegmentBase indexRange="727-770" timescale="1000">
<Initialization range="0-726"/>
</SegmentBase>
</Representation>
</AdaptationSet>
<AdaptationSet id="1" contentType="video" width="1080" height="720" frameRate="30000/1001" subsegmentAlignment="true" par="3:2">
<Representation id="1" bandwidth="721967" codecs="avc1.64001f" mimeType="video/mp4" sar="1:1">
<BaseURL>sample.video.mp4</BaseURL>
<SegmentBase indexRange="862-905" timescale="30000">
<Initialization range="0-861"/>
</SegmentBase>
</Representation>
</AdaptationSet>
</Period>
</MPD>
testdata/src/test/assets/playbackdumps/dash/webvtt-in-mp4.dump 0 → 100644
MediaCodec (video/avc):
buffers.length = 31
buffers[0] = length 36692, hash D216076E
buffers[1] = length 5312, hash D45D3CA0
buffers[2] = length 599, hash 1BE7812D
buffers[3] = length 7735, hash 4490F110
buffers[4] = length 987, hash 560B5036
buffers[5] = length 673, hash ED7CD8C7
buffers[6] = length 523, hash 3020DF50
buffers[7] = length 6061, hash 736C72B2
buffers[8] = length 992, hash FE132F23
buffers[9] = length 623, hash 5B2C1816
buffers[10] = length 421, hash 742E69C1
buffers[11] = length 4899, hash F72F86A1
buffers[12] = length 568, hash 519A8E50
buffers[13] = length 620, hash 3990AA39
buffers[14] = length 5450, hash F06EC4AA
buffers[15] = length 1051, hash 92DFA63A
buffers[16] = length 874, hash 69587FB4
buffers[17] = length 781, hash 36BE495B
buffers[18] = length 4725, hash AC0C8CD3
buffers[19] = length 1022, hash 5D8BFF34
buffers[20] = length 790, hash 99413A99
buffers[21] = length 610, hash 5E129290
buffers[22] = length 2751, hash 769974CB
buffers[23] = length 745, hash B78A477A
buffers[24] = length 621, hash CF741E7A
buffers[25] = length 505, hash 1DB4894E
buffers[26] = length 1268, hash C15348DC
buffers[27] = length 880, hash C2DE85D0
buffers[28] = length 530, hash C98BC6A8
buffers[29] = length 568, hash 4FE5C8EA
buffers[30] = length 0, hash 1
TextOutput:
Subtitle[0]:
Cues = []
Subtitle[1]:
Cue[0]:
text = This is the first subtitle.
textAlignment = ALIGN_CENTER
lineType = 1
lineAnchor = 0
position = 0.5
positionAnchor = 1
size = 1.0
Subtitle[2]:
Cues = []
Subtitle[3]:
Cue[0]:
text = This is the second subtitle.
textAlignment = ALIGN_CENTER
lineType = 1
lineAnchor = 0
position = 0.5
positionAnchor = 1
size = 1.0
Subtitle[4]:
Cues = []
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment