Skip to content

SDK / exoplayer

Go to a project
Commit 1bc316de by rohks Committed by microkatz
Options

Add timestamp to `CueGroup` 

`TextRenderer` is updated to output `CueGroup`, which contains the presentation time of the cues, in microseconds.

PiperOrigin-RevId: 456531399
(cherry picked from commit bf11a8a8)
parent 6edd1d2a
Expand all
Inline Side-by-side
Showing with 135 additions and 31 deletions
extensions/cast/src/main/java/com/google/android/exoplayer2/ext/cast/CastPlayer.java
......@@ -111,6 +111,8 @@ public final class CastPlayer extends BasePlayer {
private static final long PROGRESS_REPORT_PERIOD_MS = 1000;
private static final long[] EMPTY_TRACK_ID_ARRAY = new long[0];
private static final CueGroup EMPTY_CUE_GROUP =
new CueGroup(ImmutableList.of(), /* presentationTimeUs= */ 0);
private final CastContext castContext;
private final MediaItemConverter mediaItemConverter;
......@@ -724,7 +726,7 @@ public final class CastPlayer extends BasePlayer {
/** This method is not supported and returns an empty {@link CueGroup}. */
@Override
public CueGroup getCurrentCues() {
return CueGroup.EMPTY;
return EMPTY_CUE_GROUP;
}
/** This method is not supported and always returns {@link DeviceInfo#UNKNOWN}. */
......
library/common/src/main/java/com/google/android/exoplayer2/text/CueGroup.java
......@@ -22,6 +22,7 @@ import android.os.Bundle;
import androidx.annotation.IntDef;
import androidx.annotation.Nullable;
import com.google.android.exoplayer2.Bundleable;
import com.google.android.exoplayer2.Timeline;
import com.google.android.exoplayer2.util.BundleableUtil;
import com.google.common.collect.ImmutableList;
import java.lang.annotation.Documented;
......@@ -33,10 +34,6 @@ import java.util.List;
/** Class to represent the state of active {@link Cue Cues} at a particular time. */
public final class CueGroup implements Bundleable {
/** Empty {@link CueGroup}. */
public static final CueGroup EMPTY = new CueGroup(ImmutableList.of());
/**
* The cues in this group.
*
......@@ -46,10 +43,17 @@ public final class CueGroup implements Bundleable {
* <p>This list may be empty if the group represents a state with no cues.
*/
public final ImmutableList<Cue> cues;
/**
* The presentation time of the {@link #cues}, in microseconds.
*
* <p>This time is an offset from the start of the current {@link Timeline.Period}
*/
public final long presentationTimeUs;
/** Creates a CueGroup. */
public CueGroup(List<Cue> cues) {
public CueGroup(List<Cue> cues, long presentationTimeUs) {
this.cues = ImmutableList.copyOf(cues);
this.presentationTimeUs = presentationTimeUs;
}
// Bundleable implementation.
......@@ -57,16 +61,18 @@ public final class CueGroup implements Bundleable {
@Documented
@Retention(RetentionPolicy.SOURCE)
@Target(TYPE_USE)
@IntDef({FIELD_CUES})
@IntDef({FIELD_CUES, FIELD_PRESENTATION_TIME_US})
private @interface FieldNumber {}
private static final int FIELD_CUES = 0;
private static final int FIELD_PRESENTATION_TIME_US = 1;
@Override
public Bundle toBundle() {
Bundle bundle = new Bundle();
bundle.putParcelableArrayList(
keyForField(FIELD_CUES), BundleableUtil.toBundleArrayList(filterOutBitmapCues(cues)));
bundle.putLong(keyForField(FIELD_PRESENTATION_TIME_US), presentationTimeUs);
return bundle;
}
......@@ -78,7 +84,8 @@ public final class CueGroup implements Bundleable {
cueBundles == null
? ImmutableList.of()
: BundleableUtil.fromBundleList(Cue.CREATOR, cueBundles);
return new CueGroup(cues);
long presentationTimeUs = bundle.getLong(keyForField(FIELD_PRESENTATION_TIME_US));
return new CueGroup(cues, presentationTimeUs);
}
private static String keyForField(@FieldNumber int field) {
......
library/common/src/test/java/com/google/android/exoplayer2/text/CueGroupTest.java
......@@ -37,7 +37,7 @@ public class CueGroupTest {
Cue bitmapCue =
new Cue.Builder().setBitmap(Bitmap.createBitmap(1, 1, Bitmap.Config.ARGB_8888)).build();
ImmutableList<Cue> cues = ImmutableList.of(textCue, bitmapCue);
CueGroup cueGroup = new CueGroup(cues);
CueGroup cueGroup = new CueGroup(cues, /* presentationTimeUs= */ 1_230_000);
Parcel parcel = Parcel.obtain();
try {
......
library/core/src/main/java/com/google/android/exoplayer2/ExoPlayerImpl.java
......@@ -344,7 +344,7 @@ import java.util.concurrent.TimeoutException;
} else {
audioSessionId = Util.generateAudioSessionIdV21(applicationContext);
}
currentCueGroup = CueGroup.EMPTY;
currentCueGroup = new CueGroup(ImmutableList.of(), /* presentationTimeUs= */ 0);
throwsWhenUsingWrongThread = true;
addListener(analyticsCollector);
......@@ -931,7 +931,7 @@ import java.util.concurrent.TimeoutException;
verifyApplicationThread();
audioFocusManager.updateAudioFocus(getPlayWhenReady(), Player.STATE_IDLE);
stopInternal(reset, /* error= */ null);
currentCueGroup = CueGroup.EMPTY;
currentCueGroup = new CueGroup(ImmutableList.of(), playbackInfo.positionUs);
}
@Override
......@@ -985,7 +985,7 @@ import java.util.concurrent.TimeoutException;
checkNotNull(priorityTaskManager).remove(C.PRIORITY_PLAYBACK);
isPriorityTaskManagerRegistered = false;
}
currentCueGroup = CueGroup.EMPTY;
currentCueGroup = new CueGroup(ImmutableList.of(), /* presentationTimeUs= */ 0);
playerReleased = true;
}
......
library/core/src/main/java/com/google/android/exoplayer2/text/TextRenderer.java
......@@ -34,12 +34,13 @@ import com.google.android.exoplayer2.source.SampleStream.ReadDataResult;
import com.google.android.exoplayer2.util.Log;
import com.google.android.exoplayer2.util.MimeTypes;
import com.google.android.exoplayer2.util.Util;
import com.google.common.collect.ImmutableList;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.util.Collections;
import java.util.List;
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
import org.checkerframework.dataflow.qual.SideEffectFree;
/**
* A renderer for text.
......@@ -94,6 +95,8 @@ public final class TextRenderer extends BaseRenderer implements Callback {
@Nullable private SubtitleOutputBuffer nextSubtitle;
private int nextSubtitleEventIndex;
private long finalStreamEndPositionUs;
private long outputStreamOffsetUs;
private long lastRendererPositionUs;
/**
* @param output The output.
......@@ -125,6 +128,8 @@ public final class TextRenderer extends BaseRenderer implements Callback {
this.decoderFactory = decoderFactory;
formatHolder = new FormatHolder();
finalStreamEndPositionUs = C.TIME_UNSET;
outputStreamOffsetUs = C.TIME_UNSET;
lastRendererPositionUs = C.TIME_UNSET;
}
@Override
......@@ -161,6 +166,7 @@ public final class TextRenderer extends BaseRenderer implements Callback {
@Override
protected void onStreamChanged(Format[] formats, long startPositionUs, long offsetUs) {
outputStreamOffsetUs = offsetUs;
streamFormat = formats[0];
if (decoder != null) {
decoderReplacementState = REPLACEMENT_STATE_SIGNAL_END_OF_STREAM;
......@@ -171,6 +177,7 @@ public final class TextRenderer extends BaseRenderer implements Callback {
@Override
protected void onPositionReset(long positionUs, boolean joining) {
lastRendererPositionUs = positionUs;
clearOutput();
inputStreamEnded = false;
outputStreamEnded = false;
......@@ -185,6 +192,7 @@ public final class TextRenderer extends BaseRenderer implements Callback {
@Override
public void render(long positionUs, long elapsedRealtimeUs) {
lastRendererPositionUs = positionUs;
if (isCurrentStreamFinal()
&& finalStreamEndPositionUs != C.TIME_UNSET
&& positionUs >= finalStreamEndPositionUs) {
......@@ -248,7 +256,9 @@ public final class TextRenderer extends BaseRenderer implements Callback {
// If textRendererNeedsUpdate then subtitle must be non-null.
checkNotNull(subtitle);
// textRendererNeedsUpdate is set and we're playing. Update the renderer.
updateOutput(subtitle.getCues(positionUs));
long presentationTimeUs = getPresentationTimeUs(getCurrentEventTimeUs(positionUs));
CueGroup cueGroup = new CueGroup(subtitle.getCues(positionUs), presentationTimeUs);
updateOutput(cueGroup);
}
if (decoderReplacementState == REPLACEMENT_STATE_WAIT_END_OF_STREAM) {
......@@ -306,6 +316,8 @@ public final class TextRenderer extends BaseRenderer implements Callback {
streamFormat = null;
finalStreamEndPositionUs = C.TIME_UNSET;
clearOutput();
outputStreamOffsetUs = C.TIME_UNSET;
lastRendererPositionUs = C.TIME_UNSET;
releaseDecoder();
}
......@@ -361,33 +373,33 @@ public final class TextRenderer extends BaseRenderer implements Callback {
: subtitle.getEventTime(nextSubtitleEventIndex);
}
private void updateOutput(List<Cue> cues) {
private void updateOutput(CueGroup cueGroup) {
if (outputHandler != null) {
outputHandler.obtainMessage(MSG_UPDATE_OUTPUT, cues).sendToTarget();
outputHandler.obtainMessage(MSG_UPDATE_OUTPUT, cueGroup).sendToTarget();
} else {
invokeUpdateOutputInternal(cues);
invokeUpdateOutputInternal(cueGroup);
}
}
private void clearOutput() {
updateOutput(Collections.emptyList());
updateOutput(new CueGroup(ImmutableList.of(), getPresentationTimeUs(lastRendererPositionUs)));
}
@SuppressWarnings("unchecked")
@Override
public boolean handleMessage(Message msg) {
switch (msg.what) {
case MSG_UPDATE_OUTPUT:
invokeUpdateOutputInternal((List<Cue>) msg.obj);
invokeUpdateOutputInternal((CueGroup) msg.obj);
return true;
default:
throw new IllegalStateException();
}
}
private void invokeUpdateOutputInternal(List<Cue> cues) {
output.onCues(cues);
output.onCues(new CueGroup(cues));
@SuppressWarnings("deprecation") // We need to call both onCues method for backward compatibility.
private void invokeUpdateOutputInternal(CueGroup cueGroup) {
output.onCues(cueGroup.cues);
output.onCues(cueGroup);
}
/**
......@@ -401,4 +413,25 @@ public final class TextRenderer extends BaseRenderer implements Callback {
clearOutput();
replaceDecoder();
}
@RequiresNonNull("subtitle")
@SideEffectFree
private long getCurrentEventTimeUs(long positionUs) {
int nextEventTimeIndex = subtitle.getNextEventTimeIndex(positionUs);
if (nextEventTimeIndex == 0) {
return subtitle.timeUs;
}
return nextEventTimeIndex == C.INDEX_UNSET
? subtitle.getEventTime(subtitle.getEventTimeCount() - 1)
: subtitle.getEventTime(nextEventTimeIndex - 1);
}
@SideEffectFree
private long getPresentationTimeUs(long positionUs) {
checkState(positionUs != C.TIME_UNSET);
checkState(outputStreamOffsetUs != C.TIME_UNSET);
return positionUs - outputStreamOffsetUs;
}
}
library/core/src/test/java/com/google/android/exoplayer2/e2etest/PlaylistPlaybackTest.java
......@@ -16,17 +16,25 @@
package com.google.android.exoplayer2.e2etest;
import android.content.Context;
import android.graphics.SurfaceTexture;
import android.net.Uri;
import android.view.Surface;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.ExoPlayer;
import com.google.android.exoplayer2.MediaItem;
import com.google.android.exoplayer2.Player;
import com.google.android.exoplayer2.robolectric.PlaybackOutput;
import com.google.android.exoplayer2.robolectric.ShadowMediaCodecConfig;
import com.google.android.exoplayer2.robolectric.TestPlayerRunHelper;
import com.google.android.exoplayer2.source.DefaultMediaSourceFactory;
import com.google.android.exoplayer2.source.MediaSource;
import com.google.android.exoplayer2.testutil.CapturingRenderersFactory;
import com.google.android.exoplayer2.testutil.DumpFileAsserts;
import com.google.android.exoplayer2.testutil.FakeClock;
import com.google.android.exoplayer2.util.MimeTypes;
import com.google.common.collect.ImmutableList;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
......@@ -82,4 +90,43 @@ public final class PlaylistPlaybackTest {
DumpFileAsserts.assertOutput(
applicationContext, playbackOutput, "playbackdumps/playlists/bypass-off-then-on.dump");
}
@Test
public void test_subtitle() throws Exception {
Context applicationContext = ApplicationProvider.getApplicationContext();
CapturingRenderersFactory capturingRenderersFactory =
new CapturingRenderersFactory(applicationContext);
MediaSource.Factory mediaSourceFactory =
new DefaultMediaSourceFactory(applicationContext)
.experimentalUseProgressiveMediaSourceForSubtitles(true);
ExoPlayer player =
new ExoPlayer.Builder(applicationContext, capturingRenderersFactory)
.setClock(new FakeClock(/* isAutoAdvancing= */ true))
.setMediaSourceFactory(mediaSourceFactory)
.build();
player.setVideoSurface(new Surface(new SurfaceTexture(/* texName= */ 1)));
PlaybackOutput playbackOutput = PlaybackOutput.register(player, capturingRenderersFactory);
player.addMediaItem(MediaItem.fromUri("asset:///media/mp4/preroll-5s.mp4"));
MediaItem mediaItemWithSubtitle =
new MediaItem.Builder()
.setUri("asset:///media/mp4/preroll-5s.mp4")
.setSubtitleConfigurations(
ImmutableList.of(
new MediaItem.SubtitleConfiguration.Builder(
Uri.parse("asset:///media/webvtt/typical"))
.setMimeType(MimeTypes.TEXT_VTT)
.setLanguage("en")
.setSelectionFlags(C.SELECTION_FLAG_DEFAULT)
.build()))
.build();
player.addMediaItem(mediaItemWithSubtitle);
player.prepare();
player.play();
TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_ENDED);
player.release();
DumpFileAsserts.assertOutput(
applicationContext, playbackOutput, "playbackdumps/playlists/playlist_with_subtitles.dump");
}
}
robolectricutils/src/main/java/com/google/android/exoplayer2/robolectric/PlaybackOutput.java
......@@ -15,6 +15,8 @@
*/
package com.google.android.exoplayer2.robolectric;
import static java.lang.Math.max;
import android.graphics.Bitmap;
import androidx.annotation.Nullable;
import com.google.android.exoplayer2.ExoPlayer;
......@@ -54,7 +56,7 @@ public final class PlaybackOutput implements Dumper.Dumpable {
private final CapturingRenderersFactory capturingRenderersFactory;
private final List<Metadata> metadatas;
private final List<List<Cue>> subtitles;
private final List<CueGroup> subtitles;
private final List<List<Cue>> subtitlesFromDeprecatedTextOutput;
private PlaybackOutput(ExoPlayer player, CapturingRenderersFactory capturingRenderersFactory) {
......@@ -63,8 +65,8 @@ public final class PlaybackOutput implements Dumper.Dumpable {
metadatas = Collections.synchronizedList(new ArrayList<>());
subtitles = Collections.synchronizedList(new ArrayList<>());
subtitlesFromDeprecatedTextOutput = Collections.synchronizedList(new ArrayList<>());
// TODO: Consider passing playback position into MetadataOutput and TextOutput. Calling
// player.getCurrentPosition() inside onMetadata/Cues will likely be non-deterministic
// TODO: Consider passing playback position into MetadataOutput. Calling
// player.getCurrentPosition() inside onMetadata will likely be non-deterministic
// because renderer-thread != playback-thread.
player.addListener(
new Player.Listener() {
......@@ -80,7 +82,7 @@ public final class PlaybackOutput implements Dumper.Dumpable {
@Override
public void onCues(CueGroup cueGroup) {
subtitles.add(cueGroup.cues);
subtitles.add(cueGroup);
}
});
}
......@@ -152,9 +154,9 @@ public final class PlaybackOutput implements Dumper.Dumpable {
}
private void dumpSubtitles(Dumper dumper) {
if (!subtitles.equals(subtitlesFromDeprecatedTextOutput)) {
if (subtitles.size() != subtitlesFromDeprecatedTextOutput.size()) {
throw new IllegalStateException(
"Expected subtitles to be equal from both implementations of onCues method.");
"Expected subtitles to be of equal length from both implementations of onCues method.");
}
if (subtitles.isEmpty()) {
......@@ -163,7 +165,15 @@ public final class PlaybackOutput implements Dumper.Dumpable {
dumper.startBlock("TextOutput");
for (int i = 0; i < subtitles.size(); i++) {
dumper.startBlock("Subtitle[" + i + "]");
List<Cue> subtitle = subtitles.get(i);
// TODO: Solving https://github.com/google/ExoPlayer/issues/9672 will allow us to remove this
// hack of forcing presentationTimeUs to be >= 0.
dumper.add("presentationTimeUs", max(0, subtitles.get(i).presentationTimeUs));
ImmutableList<Cue> subtitle = subtitles.get(i).cues;
if (!subtitle.equals(subtitlesFromDeprecatedTextOutput.get(i))) {
throw new IllegalStateException(
"Expected subtitle to be equal from both implementations of onCues method for index "
+ i);
}
if (subtitle.isEmpty()) {
dumper.add("Cues", ImmutableList.of());
}
......
testdata/src/test/assets/playbackdumps/webvtt/typical.dump
......@@ -348,8 +348,10 @@ MediaCodecAdapter (exotest.video.avc):
buffers[125] = length 0, hash 1
TextOutput:
Subtitle[0]:
presentationTimeUs = 0
Cues = []
Subtitle[1]:
presentationTimeUs = 0
Cue[0]:
text = This is the first subtitle.
textAlignment = ALIGN_CENTER
......@@ -360,8 +362,10 @@ TextOutput:
positionAnchor = 1
size = 1.0
Subtitle[2]:
presentationTimeUs = 1234000
Cues = []
Subtitle[3]:
presentationTimeUs = 2345000
Cue[0]:
text = This is the second subtitle.
textAlignment = ALIGN_CENTER
......@@ -372,4 +376,5 @@ TextOutput:
positionAnchor = 1
size = 1.0
Subtitle[4]:
presentationTimeUs = 3456000
Cues = []
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment