#1583 - Parsing base64 encoded image data from metadata

254a586e · Arnold Szabo · 34797651 · 254a586e · 254a586e · 254a586e
Commit 254a586e authored Nov 06, 2018 by Arnold Szabo
Showing with 95 additions and 14 deletions
demos/main/src/main/assets/media.exolist.json
library/core/src/main/java/com/google/android/exoplayer2/text/ttml/TtmlDecoder.java
library/core/src/main/java/com/google/android/exoplayer2/text/ttml/TtmlNode.java
library/core/src/main/java/com/google/android/exoplayer2/text/ttml/TtmlSubtitle.java
--- a/demos/main/src/main/assets/media.exolist.json
+++ b/demos/main/src/main/assets/media.exolist.json
@@ -3,6 +3,11 @@
    "name": "YouTube DASH",
    "samples": [
      {
+        "name": "DVB Image sub",
+        "uri": "https://livesim.dashif.org/dash/vod/testpic_2s/img_subs.mpd",
+        "extension": "mpd"
+      },
+      {
        "name": "Google Glass (MP4,H264)",
        "uri": "https://www.youtube.com/api/manifest/dash/id/bf5bb2419360daf1/source/youtube?as=fmp4_audio_clear,fmp4_sd_hd_clear&sparams=ip,ipbits,expire,source,id,as&ip=0.0.0.0&ipbits=0&expire=19000000000&signature=51AF5F39AB0CEC3E5497CD9C900EBFEAECCCB5C7.8506521BFC350652163895D4C26DEE124209AA9E&key=ik0",
        "extension": "mpd"

--- a/library/core/src/main/java/com/google/android/exoplayer2/text/ttml/TtmlDecoder.java
+++ b/library/core/src/main/java/com/google/android/exoplayer2/text/ttml/TtmlDecoder.java
@@ -68,6 +68,8 @@ public final class TtmlDecoder extends SimpleSubtitleDecoder {
  private static final String ATTR_END = "end";
  private static final String ATTR_STYLE = "style";
  private static final String ATTR_REGION = "region";
+  private static final String ATTR_IMAGE = "backgroundImage";
  private static final Pattern CLOCK_TIME =
      Pattern.compile("^([0-9][0-9]+):([0-9][0-9]):([0-9][0-9])"
@@ -105,6 +107,7 @@ public final class TtmlDecoder extends SimpleSubtitleDecoder {
      XmlPullParser xmlParser = xmlParserFactory.newPullParser();
      Map<String, TtmlStyle> globalStyles = new HashMap<>();
      Map<String, TtmlRegion> regionMap = new HashMap<>();
+      Map<String, String> imageMap = new HashMap<>();
      regionMap.put(TtmlNode.ANONYMOUS_REGION_ID, new TtmlRegion(null));
      ByteArrayInputStream inputStream = new ByteArrayInputStream(bytes, 0, length);
      xmlParser.setInput(inputStream, null);
@@ -127,7 +130,7 @@ public final class TtmlDecoder extends SimpleSubtitleDecoder {
              Log.i(TAG, "Ignoring unsupported tag: " + xmlParser.getName());
              unsupportedNodeDepth++;
            } else if (TtmlNode.TAG_HEAD.equals(name)) {
-              parseHeader(xmlParser, globalStyles, regionMap, cellResolution);
+              parseHeader(xmlParser, globalStyles, regionMap, cellResolution, imageMap);
            } else {
              try {
                TtmlNode node = parseNode(xmlParser, parent, regionMap, frameAndTickRate);
@@ -145,7 +148,7 @@ public final class TtmlDecoder extends SimpleSubtitleDecoder {
            parent.addChild(TtmlNode.buildTextNode(xmlParser.getText()));
          } else if (eventType == XmlPullParser.END_TAG) {
            if (xmlParser.getName().equals(TtmlNode.TAG_TT)) {
-              ttmlSubtitle = new TtmlSubtitle(nodeStack.peek(), globalStyles, regionMap);
+              ttmlSubtitle = new TtmlSubtitle(nodeStack.peek(), globalStyles, regionMap, imageMap);
            }
            nodeStack.pop();
          }
@@ -230,7 +233,8 @@ public final class TtmlDecoder extends SimpleSubtitleDecoder {
      XmlPullParser xmlParser,
      Map<String, TtmlStyle> globalStyles,
      Map<String, TtmlRegion> globalRegions,
-      CellResolution cellResolution)
+      CellResolution cellResolution,
+      Map<String, String> imageMap)
      throws IOException, XmlPullParserException {
    do {
      xmlParser.next();
@@ -250,11 +254,29 @@ public final class TtmlDecoder extends SimpleSubtitleDecoder {
        if (ttmlRegion != null) {
          globalRegions.put(ttmlRegion.id, ttmlRegion);
        }
+      } else if(XmlPullParserUtil.isStartTag(xmlParser, TtmlNode.TAG_METADATA)){
+        parseMetaData(xmlParser, imageMap);
      }
    } while (!XmlPullParserUtil.isEndTag(xmlParser, TtmlNode.TAG_HEAD));
    return globalStyles;
  }
+  public void parseMetaData(XmlPullParser xmlParser, Map<String, String> imageMap) throws IOException, XmlPullParserException {
+    do {
+      xmlParser.next();
+      if (XmlPullParserUtil.isStartTag(xmlParser, TtmlNode.TAG_SMPTE_IMAGE)) {
+        for (int i = 0; i < xmlParser.getAttributeCount(); i++) {
+          String id = XmlPullParserUtil.getAttributeValue(xmlParser, "id");
+          if(id != null){
+            String base64 = xmlParser.nextText();
+            imageMap.put(id, base64);
+          }
+        }
+      }
+    } while (!XmlPullParserUtil.isEndTag(xmlParser, TtmlNode.TAG_METADATA));
+  }
  /**
   * Parses a region declaration.
   *
@@ -457,6 +479,7 @@ public final class TtmlDecoder extends SimpleSubtitleDecoder {
    long startTime = C.TIME_UNSET;
    long endTime = C.TIME_UNSET;
    String regionId = TtmlNode.ANONYMOUS_REGION_ID;
+    String imageId = "";
    String[] styleIds = null;
    int attributeCount = parser.getAttributeCount();
    TtmlStyle style = parseStyleAttributes(parser, null);
@@ -487,6 +510,9 @@ public final class TtmlDecoder extends SimpleSubtitleDecoder {
            regionId = value;
          }
          break;
+        case ATTR_IMAGE:
+          imageId = value.substring(1);
+          break;
        default:
          // Do nothing.
          break;
@@ -509,7 +535,7 @@ public final class TtmlDecoder extends SimpleSubtitleDecoder {
        endTime = parent.endTimeUs;
      }
    }
-    return TtmlNode.buildNode(parser.getName(), startTime, endTime, style, styleIds, regionId);
+    return TtmlNode.buildNode(parser.getName(), startTime, endTime, style, styleIds, regionId, imageId);
  }
  private static boolean isSupportedTag(String tag) {

--- a/library/core/src/main/java/com/google/android/exoplayer2/text/ttml/TtmlNode.java
+++ b/library/core/src/main/java/com/google/android/exoplayer2/text/ttml/TtmlNode.java
@@ -15,10 +15,16 @@
 */
 package com.google.android.exoplayer2.text.ttml;
+import android.graphics.Bitmap;
+import android.graphics.BitmapFactory;
 import android.text.SpannableStringBuilder;
+import android.util.Base64;
+import android.util.Pair;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.text.Cue;
 import com.google.android.exoplayer2.util.Assertions;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -44,9 +50,9 @@ import java.util.TreeSet;
  public static final String TAG_LAYOUT = "layout";
  public static final String TAG_REGION = "region";
  public static final String TAG_METADATA = "metadata";
-  public static final String TAG_SMPTE_IMAGE = "smpte:image";
+  public static final String TAG_SMPTE_IMAGE = "image";
-  public static final String TAG_SMPTE_DATA = "smpte:data";
+  public static final String TAG_SMPTE_DATA = "data";
-  public static final String TAG_SMPTE_INFORMATION = "smpte:information";
+  public static final String TAG_SMPTE_INFORMATION = "information";
  public static final String ANONYMOUS_REGION_ID = "";
  public static final String ATTR_ID = "id";
@@ -82,6 +88,7 @@ import java.util.TreeSet;
  public final long endTimeUs;
  public final TtmlStyle style;
  public final String regionId;
+  public final String imageId;
  private final String[] styleIds;
  private final HashMap<String, Integer> nodeStartsByRegion;
@@ -91,18 +98,19 @@ import java.util.TreeSet;
  public static TtmlNode buildTextNode(String text) {
    return new TtmlNode(null, TtmlRenderUtil.applyTextElementSpacePolicy(text), C.TIME_UNSET,
-        C.TIME_UNSET, null, null, ANONYMOUS_REGION_ID);
+        C.TIME_UNSET, null, null, ANONYMOUS_REGION_ID, null);
  }
  public static TtmlNode buildNode(String tag, long startTimeUs, long endTimeUs,
-      TtmlStyle style, String[] styleIds, String regionId) {
+                                   TtmlStyle style, String[] styleIds, String regionId, String imageId) {
-    return new TtmlNode(tag, null, startTimeUs, endTimeUs, style, styleIds, regionId);
+    return new TtmlNode(tag, null, startTimeUs, endTimeUs, style, styleIds, regionId, imageId);
  }
  private TtmlNode(String tag, String text, long startTimeUs, long endTimeUs,
-      TtmlStyle style, String[] styleIds, String regionId) {
+                   TtmlStyle style, String[] styleIds, String regionId, String imageId) {
    this.tag = tag;
    this.text = text;
+    this.imageId = imageId;
    this.style = style;
    this.styleIds = styleIds;
    this.isTextNode = text != null;
@@ -172,11 +180,37 @@ import java.util.TreeSet;
  }
  public List<Cue> getCues(long timeUs, Map<String, TtmlStyle> globalStyles,
-      Map<String, TtmlRegion> regionMap) {
+                           Map<String, TtmlRegion> regionMap, Map<String, String> imageMap) {
    TreeMap<String, SpannableStringBuilder> regionOutputs = new TreeMap<>();
+    List<Pair<String, String>> regionImageList = new ArrayList<>();
    traverseForText(timeUs, false, regionId, regionOutputs);
    traverseForStyle(timeUs, globalStyles, regionOutputs);
+    traverseForImage(timeUs, regionId, regionImageList);
    List<Cue> cues = new ArrayList<>();
+    // Create text based cues
+    for (Pair<String, String> regionImagePair : regionImageList) {
+      String base64 = imageMap.get(regionImagePair.second);
+      byte[] decodedString = Base64.decode(base64, Base64.DEFAULT);
+      Bitmap decodedByte = BitmapFactory.decodeByteArray(decodedString, 0, decodedString.length);
+      TtmlRegion region = regionMap.get(regionImagePair.first);
+      cues.add(
+          new Cue(decodedByte,
+              region.position,
+              Cue.TYPE_UNSET,
+              region.line,
+              region.lineAnchor,
+              region.width,
+              Cue.DIMEN_UNSET
+          )
+      );
+    }
+    // Create image based cues
    for (Entry<String, SpannableStringBuilder> entry : regionOutputs.entrySet()) {
      TtmlRegion region = regionMap.get(entry.getKey());
      cues.add(
@@ -195,6 +229,19 @@ import java.util.TreeSet;
    return cues;
  }
+  private void traverseForImage(long timeUs, String inheritedRegion, List<Pair<String, String>> regionImageList) {
+      // TODO isActive needed?
+    String resolvedRegionId = ANONYMOUS_REGION_ID.equals(regionId) ? inheritedRegion : regionId;
+    if (TAG_DIV.equals(tag) && imageId != null) {
+      regionImageList.add(new Pair<>(resolvedRegionId, imageId));
+    }
+    for (int i = 0; i < getChildCount(); ++i) {
+      getChild(i).traverseForImage(timeUs, resolvedRegionId, regionImageList);
+    }
+  }
  private void traverseForText(
      long timeUs,
      boolean descendsPNode,

--- a/library/core/src/main/java/com/google/android/exoplayer2/text/ttml/TtmlSubtitle.java
+++ b/library/core/src/main/java/com/google/android/exoplayer2/text/ttml/TtmlSubtitle.java
@@ -32,11 +32,14 @@ import java.util.Map;
  private final long[] eventTimesUs;
  private final Map<String, TtmlStyle> globalStyles;
  private final Map<String, TtmlRegion> regionMap;
+  private final Map<String, String> imageMap;
  public TtmlSubtitle(TtmlNode root, Map<String, TtmlStyle> globalStyles,
-      Map<String, TtmlRegion> regionMap) {
+      Map<String, TtmlRegion> regionMap, Map<String, String> imageMap) {
    this.root = root;
    this.regionMap = regionMap;
+    this.imageMap = imageMap;
    this.globalStyles =
        globalStyles != null ? Collections.unmodifiableMap(globalStyles) : Collections.emptyMap();
    this.eventTimesUs = root.getEventTimesUs();
@@ -65,7 +68,7 @@ import java.util.Map;
  @Override
  public List<Cue> getCues(long timeUs) {
-    return root.getCues(timeUs, globalStyles, regionMap);
+    return root.getCues(timeUs, globalStyles, regionMap, imageMap);
  }
  /* @VisibleForTesting */