Skip to content

SDK / exoplayer

Go to a project
Commit eb51ad57 by Ian Baker Committed by christosts
Options

Merge pull request #10750 from Stronger197:subrip_utf_16 

PiperOrigin-RevId: 492164739
(cherry picked from commit 496cfa42)
parent 2bfced9b
Expand all
Inline Side-by-side
Showing with 182 additions and 40 deletions
library/common/src/main/java/com/google/android/exoplayer2/util/ParsableByteArray.java
......@@ -17,6 +17,9 @@ package com.google.android.exoplayer2.util;
import androidx.annotation.Nullable;
import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableSet;
import com.google.common.primitives.Chars;
import com.google.common.primitives.UnsignedBytes;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.util.Arrays;
......@@ -27,6 +30,12 @@ import java.util.Arrays;
*/
public final class ParsableByteArray {
private static final char[] CR_AND_LF = {'\r', '\n'};
private static final char[] LF = {'\n'};
private static final ImmutableSet<Charset> SUPPORTED_CHARSETS_FOR_READLINE =
ImmutableSet.of(
Charsets.US_ASCII, Charsets.UTF_8, Charsets.UTF_16, Charsets.UTF_16BE, Charsets.UTF_16LE);
private byte[] data;
private int position;
// TODO(internal b/147657250): Enforce this limit on all read methods.
......@@ -489,45 +498,47 @@ public final class ParsableByteArray {
}
/**
* Reads a line of text.
* Reads a line of text in UTF-8.
*
* <p>Equivalent to passing {@link Charsets#UTF_8} to {@link #readLine(Charset)}.
*/
@Nullable
public String readLine() {
return readLine(Charsets.UTF_8);
}
/**
* Reads a line of text in {@code charset}.
*
* <p>A line is considered to be terminated by any one of a carriage return ('\r'), a line feed
* ('\n'), or a carriage return followed immediately by a line feed ('\r\n'). The UTF-8 charset is
* used. This method discards leading UTF-8 byte order marks, if present.
* ('\n'), or a carriage return followed immediately by a line feed ('\r\n'). This method discards
* leading UTF byte order marks (BOM), if present.
*
* <p>The {@linkplain #getPosition() position} is advanced to start of the next line (i.e. any
* line terminators are skipped).
*
* @param charset The charset used to interpret the bytes as a {@link String}.
* @return The line not including any line-termination characters, or null if the end of the data
* has already been reached.
* @throws IllegalArgumentException if charset is not supported. Only US_ASCII, UTF-8, UTF-16,
* UTF-16BE, and UTF-16LE are supported.
*/
@Nullable
public String readLine() {
public String readLine(Charset charset) {
Assertions.checkArgument(
SUPPORTED_CHARSETS_FOR_READLINE.contains(charset), "Unsupported charset: " + charset);
if (bytesLeft() == 0) {
return null;
}
int lineLimit = position;
while (lineLimit < limit && !Util.isLinebreak(data[lineLimit])) {
lineLimit++;
if (!charset.equals(Charsets.US_ASCII)) {
readUtfCharsetFromBom(); // Skip BOM if present
}
if (lineLimit - position >= 3
&& data[position] == (byte) 0xEF
&& data[position + 1] == (byte) 0xBB
&& data[position + 2] == (byte) 0xBF) {
// There's a UTF-8 byte order mark at the start of the line. Discard it.
position += 3;
}
String line = Util.fromUtf8Bytes(data, position, lineLimit - position);
position = lineLimit;
int lineLimit = findNextLineTerminator(charset);
String line = readString(lineLimit - position, charset);
if (position == limit) {
return line;
}
if (data[position] == '\r') {
position++;
if (position == limit) {
return line;
}
}
if (data[position] == '\n') {
position++;
}
skipLineTerminator(charset);
return line;
}
......@@ -565,4 +576,99 @@ public final class ParsableByteArray {
position += length;
return value;
}
/**
* Reads a UTF byte order mark (BOM) and returns the UTF {@link Charset} it represents. Returns
* {@code null} without advancing {@link #getPosition() position} if no BOM is found.
*/
@Nullable
public Charset readUtfCharsetFromBom() {
if (bytesLeft() >= 3
&& data[position] == (byte) 0xEF
&& data[position + 1] == (byte) 0xBB
&& data[position + 2] == (byte) 0xBF) {
position += 3;
return Charsets.UTF_8;
} else if (bytesLeft() >= 2) {
if (data[position] == (byte) 0xFE && data[position + 1] == (byte) 0xFF) {
position += 2;
return Charsets.UTF_16BE;
} else if (data[position] == (byte) 0xFF && data[position + 1] == (byte) 0xFE) {
position += 2;
return Charsets.UTF_16LE;
}
}
return null;
}
/**
* Returns the index of the next occurrence of '\n' or '\r', or {@link #limit} if none is found.
*/
private int findNextLineTerminator(Charset charset) {
int stride;
if (charset.equals(Charsets.UTF_8) || charset.equals(Charsets.US_ASCII)) {
stride = 1;
} else if (charset.equals(Charsets.UTF_16)
|| charset.equals(Charsets.UTF_16LE)
|| charset.equals(Charsets.UTF_16BE)) {
stride = 2;
} else {
throw new IllegalArgumentException("Unsupported charset: " + charset);
}
for (int i = position; i < limit - (stride - 1); i += stride) {
if ((charset.equals(Charsets.UTF_8) || charset.equals(Charsets.US_ASCII))
&& Util.isLinebreak(data[i])) {
return i;
} else if ((charset.equals(Charsets.UTF_16) || charset.equals(Charsets.UTF_16BE))
&& data[i] == 0x00
&& Util.isLinebreak(data[i + 1])) {
return i;
} else if (charset.equals(Charsets.UTF_16LE)
&& data[i + 1] == 0x00
&& Util.isLinebreak(data[i])) {
return i;
}
}
return limit;
}
private void skipLineTerminator(Charset charset) {
if (readCharacterIfInList(charset, CR_AND_LF) == '\r') {
readCharacterIfInList(charset, LF);
}
}
/**
* Peeks at the character at {@link #position} (as decoded by {@code charset}), returns it and
* advances {@link #position} past it if it's in {@code chars}, otherwise returns {@code 0}
* without advancing {@link #position}. Returns {@code 0} if {@link #bytesLeft()} doesn't allow
* reading a whole character in {@code charset}.
*
* <p>Only supports characters in {@code chars} that occupy a single code unit (i.e. one byte for
* UTF-8 and two bytes for UTF-16).
*/
private char readCharacterIfInList(Charset charset, char[] chars) {
char character;
int characterSize;
if ((charset.equals(Charsets.UTF_8) || charset.equals(Charsets.US_ASCII)) && bytesLeft() >= 1) {
character = Chars.checkedCast(UnsignedBytes.toInt(data[position]));
characterSize = 1;
} else if ((charset.equals(Charsets.UTF_16) || charset.equals(Charsets.UTF_16BE))
&& bytesLeft() >= 2) {
character = Chars.fromBytes(data[position], data[position + 1]);
characterSize = 2;
} else if (charset.equals(Charsets.UTF_16LE) && bytesLeft() >= 2) {
character = Chars.fromBytes(data[position + 1], data[position]);
characterSize = 2;
} else {
return 0;
}
if (Chars.contains(chars, character)) {
position += characterSize;
return Chars.checkedCast(character);
} else {
return 0;
}
}
}
library/extractor/src/main/java/com/google/android/exoplayer2/text/subrip/SubripDecoder.java
......@@ -26,6 +26,8 @@ import com.google.android.exoplayer2.util.Assertions;
import com.google.android.exoplayer2.util.Log;
import com.google.android.exoplayer2.util.LongArray;
import com.google.android.exoplayer2.util.ParsableByteArray;
import com.google.common.base.Charsets;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
......@@ -74,9 +76,10 @@ public final class SubripDecoder extends SimpleSubtitleDecoder {
ArrayList<Cue> cues = new ArrayList<>();
LongArray cueTimesUs = new LongArray();
ParsableByteArray subripData = new ParsableByteArray(data, length);
Charset charset = detectUtfCharset(subripData);
@Nullable String currentLine;
while ((currentLine = subripData.readLine()) != null) {
while ((currentLine = subripData.readLine(charset)) != null) {
if (currentLine.length() == 0) {
// Skip blank lines.
continue;
......@@ -91,7 +94,7 @@ public final class SubripDecoder extends SimpleSubtitleDecoder {
}
// Read and parse the timing line.
currentLine = subripData.readLine();
currentLine = subripData.readLine(charset);
if (currentLine == null) {
Log.w(TAG, "Unexpected end");
break;
......@@ -109,13 +112,13 @@ public final class SubripDecoder extends SimpleSubtitleDecoder {
// Read and parse the text and tags.
textBuilder.setLength(0);
tags.clear();
currentLine = subripData.readLine();
currentLine = subripData.readLine(charset);
while (!TextUtils.isEmpty(currentLine)) {
if (textBuilder.length() > 0) {
textBuilder.append("<br>");
}
textBuilder.append(processLine(currentLine, tags));
currentLine = subripData.readLine();
currentLine = subripData.readLine(charset);
}
Spanned text = Html.fromHtml(textBuilder.toString());
......@@ -139,6 +142,15 @@ public final class SubripDecoder extends SimpleSubtitleDecoder {
}
/**
* Determine UTF encoding of the byte array from a byte order mark (BOM), defaulting to UTF-8 if
* no BOM is found.
*/
private Charset detectUtfCharset(ParsableByteArray data) {
@Nullable Charset charset = data.readUtfCharsetFromBom();
return charset != null ? charset : Charsets.UTF_8;
}
/**
* Trims and removes tags from the given line. The removed tags are added to {@code tags}.
*
* @param line The line to process.
......
library/extractor/src/main/java/com/google/android/exoplayer2/text/tx3g/Tx3gDecoder.java
......@@ -26,6 +26,7 @@ import android.text.style.ForegroundColorSpan;
import android.text.style.StyleSpan;
import android.text.style.TypefaceSpan;
import android.text.style.UnderlineSpan;
import androidx.annotation.Nullable;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.text.Cue;
import com.google.android.exoplayer2.text.SimpleSubtitleDecoder;
......@@ -35,6 +36,7 @@ import com.google.android.exoplayer2.util.Log;
import com.google.android.exoplayer2.util.ParsableByteArray;
import com.google.android.exoplayer2.util.Util;
import com.google.common.base.Charsets;
import java.nio.charset.Charset;
import java.util.List;
/**
......@@ -46,16 +48,12 @@ public final class Tx3gDecoder extends SimpleSubtitleDecoder {
private static final String TAG = "Tx3gDecoder";
private static final char BOM_UTF16_BE = '\uFEFF';
private static final char BOM_UTF16_LE = '\uFFFE';
private static final int TYPE_STYL = 0x7374796c;
private static final int TYPE_TBOX = 0x74626f78;
private static final String TX3G_SERIF = "Serif";
private static final int SIZE_ATOM_HEADER = 8;
private static final int SIZE_SHORT = 2;
private static final int SIZE_BOM_UTF16 = 2;
private static final int SIZE_STYLE_RECORD = 12;
private static final int FONT_FACE_BOLD = 0x0001;
......@@ -171,13 +169,11 @@ public final class Tx3gDecoder extends SimpleSubtitleDecoder {
if (textLength == 0) {
return "";
}
if (parsableByteArray.bytesLeft() >= SIZE_BOM_UTF16) {
char firstChar = parsableByteArray.peekChar();
if (firstChar == BOM_UTF16_BE || firstChar == BOM_UTF16_LE) {
return parsableByteArray.readString(textLength, Charsets.UTF_16);
}
}
return parsableByteArray.readString(textLength, Charsets.UTF_8);
int textStartPosition = parsableByteArray.getPosition();
@Nullable Charset charset = parsableByteArray.readUtfCharsetFromBom();
int bomSize = parsableByteArray.getPosition() - textStartPosition;
return parsableByteArray.readString(
textLength - bomSize, charset != null ? charset : Charsets.UTF_8);
}
private void applyStyleRecord(ParsableByteArray parsableByteArray, SpannableStringBuilder cueText)
......
library/extractor/src/test/java/com/google/android/exoplayer2/text/subrip/SubripDecoderTest.java
......@@ -40,6 +40,8 @@ public final class SubripDecoderTest {
private static final String TYPICAL_NEGATIVE_TIMESTAMPS =
"media/subrip/typical_negative_timestamps";
private static final String TYPICAL_UNEXPECTED_END = "media/subrip/typical_unexpected_end";
private static final String TYPICAL_UTF16BE = "media/subrip/typical_utf16be";
private static final String TYPICAL_UTF16LE = "media/subrip/typical_utf16le";
private static final String TYPICAL_WITH_TAGS = "media/subrip/typical_with_tags";
private static final String TYPICAL_NO_HOURS_AND_MILLIS =
"media/subrip/typical_no_hours_and_millis";
......@@ -149,6 +151,32 @@ public final class SubripDecoderTest {
}
@Test
public void decodeTypicalUtf16LittleEndian() throws IOException {
SubripDecoder decoder = new SubripDecoder();
byte[] bytes =
TestUtil.getByteArray(ApplicationProvider.getApplicationContext(), TYPICAL_UTF16LE);
Subtitle subtitle = decoder.decode(bytes, bytes.length, false);
assertThat(subtitle.getEventTimeCount()).isEqualTo(6);
assertTypicalCue1(subtitle, 0);
assertTypicalCue2(subtitle, 2);
assertTypicalCue3(subtitle, 4);
}
@Test
public void decodeTypicalUtf16BigEndian() throws IOException {
SubripDecoder decoder = new SubripDecoder();
byte[] bytes =
TestUtil.getByteArray(ApplicationProvider.getApplicationContext(), TYPICAL_UTF16BE);
Subtitle subtitle = decoder.decode(bytes, bytes.length, false);
assertThat(subtitle.getEventTimeCount()).isEqualTo(6);
assertTypicalCue1(subtitle, 0);
assertTypicalCue2(subtitle, 2);
assertTypicalCue3(subtitle, 4);
}
@Test
public void decodeCueWithTag() throws IOException {
SubripDecoder decoder = new SubripDecoder();
byte[] bytes =
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment