Add support for RTSP MPEG4

Added MPEG4 RTP packet reader and added support for MPEG4
playback through RTSP

Change-Id: I57c9a61b18471dbd2c368177ebfb89ee662f995b

libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java

@@ -31,6 +31,13 @@ public final class CodecSpecificDataUtil {
   private static final String[] HEVC_GENERAL_PROFILE_SPACE_STRINGS =
   private static final String[] HEVC_GENERAL_PROFILE_SPACE_STRINGS =
       new String[] {"", "A", "B", "C"};
       new String[] {"", "A", "B", "C"};
 
 
+  // MP4V-ES
+  private static final int VISUAL_OBJECT_LAYER = 1;
+  private static final int VISUAL_OBJECT_LAYER_START = 0x20;
+  private static final int EXTENDED_PAR = 0x0F;
+  private static final int RECTANGULAR = 0x00;
+  private static final int FINE_GRANULARITY_SCALABLE = 0x12;
+
   /**
   /**
    * Parses an ALAC AudioSpecificConfig (i.e. an <a
    * Parses an ALAC AudioSpecificConfig (i.e. an <a
    * href="https://github.com/macosforge/alac/blob/master/ALACMagicCookieDescription.txt">ALACSpecificConfig</a>).
    * href="https://github.com/macosforge/alac/blob/master/ALACMagicCookieDescription.txt">ALACSpecificConfig</a>).
@@ -73,6 +80,85 @@ public final class CodecSpecificDataUtil {
   }
   }
 
 
   /**
   /**
+   * Parses an MPEG-4 Visual configuration information, as defined in ISO/IEC14496-2
+   *
+   * @param videoSpecificConfig A byte array containing the MPEG-4 Visual configuration information
+   *     to parse.
+   * @return A pair consisting of the width and the height.
+   */
+  public static Pair<Integer, Integer> parseMpeg4VideoSpecificConfig(byte[] videoSpecificConfig) {
+    int offset = 0;
+    boolean foundVOL = false;
+    ParsableByteArray scdScratchBytes = new ParsableByteArray(videoSpecificConfig);
+    while (offset + 3 < videoSpecificConfig.length) {
+      if (scdScratchBytes.readUnsignedInt24() != VISUAL_OBJECT_LAYER
+          || (videoSpecificConfig[offset + 3] & 0xf0) != VISUAL_OBJECT_LAYER_START) {
+        scdScratchBytes.setPosition(scdScratchBytes.getPosition() - 2);
+        offset++;
+        continue;
+      }
+      foundVOL = true;
+      break;
+    }
+
+    Assertions.checkArgument(foundVOL);
+
+    ParsableBitArray scdScratchBits = new ParsableBitArray(videoSpecificConfig);
+    scdScratchBits.skipBits((offset + 4) * 8);
+    scdScratchBits.skipBits(1); // random_accessible_vol
+
+    int videoObjectTypeIndication = scdScratchBits.readBits(8);
+    Assertions.checkArgument(videoObjectTypeIndication != FINE_GRANULARITY_SCALABLE);
+
+    if (scdScratchBits.readBit()) { // object_layer_identifier
+      scdScratchBits.skipBits(4); // video_object_layer_verid
+      scdScratchBits.skipBits(3); // video_object_layer_priority
+    }
+
+    int aspectRatioInfo = scdScratchBits.readBits(4);
+    if (aspectRatioInfo == EXTENDED_PAR) {
+      scdScratchBits.skipBits(8); // par_width
+      scdScratchBits.skipBits(8); // par_height
+    }
+
+    if (scdScratchBits.readBit()) { // vol_control_parameters
+      scdScratchBits.skipBits(2); // chroma_format
+      scdScratchBits.skipBits(1); // low_delay
+      if (scdScratchBits.readBit()) { // vbv_parameters
+        scdScratchBits.skipBits(79);
+      }
+    }
+
+    int videoObjectLayerShape = scdScratchBits.readBits(2);
+    Assertions.checkArgument(videoObjectLayerShape == RECTANGULAR);
+
+    Assertions.checkArgument(scdScratchBits.readBit()); // marker_bit
+    int vopTimeIncrementResolution = scdScratchBits.readBits(16);
+    Assertions.checkArgument(scdScratchBits.readBit()); // marker_bit
+
+    if (scdScratchBits.readBit()) { // fixed_vop_rate
+      Assertions.checkArgument(vopTimeIncrementResolution > 0);
+      --vopTimeIncrementResolution;
+      int numBits = 0;
+      while (vopTimeIncrementResolution > 0) {
+        ++numBits;
+        vopTimeIncrementResolution >>= 1;
+      }
+      scdScratchBits.skipBits(numBits); // fixed_vop_time_increment
+    }
+
+    Assertions.checkArgument(scdScratchBits.readBit()); // marker_bit
+    int videoObjectLayerWidth = scdScratchBits.readBits(13);
+    Assertions.checkArgument(scdScratchBits.readBit()); // marker_bit
+    int videoObjectLayerHeight = scdScratchBits.readBits(13);
+    Assertions.checkArgument(scdScratchBits.readBit()); // marker_bit
+
+    scdScratchBits.skipBits(1); // interlaced
+
+    return Pair.create(videoObjectLayerWidth, videoObjectLayerHeight);
+  }
+
+  /**
    * Builds an RFC 6381 AVC codec string using the provided parameters.
    * Builds an RFC 6381 AVC codec string using the provided parameters.
    *
    *
    * @param profileIdc The encoding profile.
    * @param profileIdc The encoding profile.

libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java

@@ -38,6 +38,7 @@ public final class RtpPayloadFormat {
 
 
   private static final String RTP_MEDIA_AC3 = "AC3";
   private static final String RTP_MEDIA_AC3 = "AC3";
   private static final String RTP_MEDIA_MPEG4_GENERIC = "MPEG4-GENERIC";
   private static final String RTP_MEDIA_MPEG4_GENERIC = "MPEG4-GENERIC";
+  private static final String RTP_MEDIA_MPEG4_VIDEO = "MP4V-ES";
   private static final String RTP_MEDIA_H264 = "H264";
   private static final String RTP_MEDIA_H264 = "H264";
 
 
   /** Returns whether the format of a {@link MediaDescription} is supported. */
   /** Returns whether the format of a {@link MediaDescription} is supported. */
@@ -45,6 +46,7 @@ public final class RtpPayloadFormat {
     switch (Ascii.toUpperCase(mediaDescription.rtpMapAttribute.mediaEncoding)) {
     switch (Ascii.toUpperCase(mediaDescription.rtpMapAttribute.mediaEncoding)) {
       case RTP_MEDIA_AC3:
       case RTP_MEDIA_AC3:
       case RTP_MEDIA_H264:
       case RTP_MEDIA_H264:
+      case RTP_MEDIA_MPEG4_VIDEO:
       case RTP_MEDIA_MPEG4_GENERIC:
       case RTP_MEDIA_MPEG4_GENERIC:
         return true;
         return true;
       default:
       default:
@@ -65,6 +67,8 @@ public final class RtpPayloadFormat {
         return MimeTypes.AUDIO_AC3;
         return MimeTypes.AUDIO_AC3;
       case RTP_MEDIA_H264:
       case RTP_MEDIA_H264:
         return MimeTypes.VIDEO_H264;
         return MimeTypes.VIDEO_H264;
+      case RTP_MEDIA_MPEG4_VIDEO:
+        return MimeTypes.VIDEO_MP4V;
       case RTP_MEDIA_MPEG4_GENERIC:
       case RTP_MEDIA_MPEG4_GENERIC:
         return MimeTypes.AUDIO_AAC;
         return MimeTypes.AUDIO_AAC;
       default:
       default:

libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java

@@ -25,6 +25,7 @@ import static androidx.media3.extractor.NalUnitUtil.NAL_START_CODE;
 
 
 import android.net.Uri;
 import android.net.Uri;
 import android.util.Base64;
 import android.util.Base64;
+import android.util.Pair;
 import androidx.annotation.Nullable;
 import androidx.annotation.Nullable;
 import androidx.annotation.VisibleForTesting;
 import androidx.annotation.VisibleForTesting;
 import androidx.media3.common.C;
 import androidx.media3.common.C;
@@ -44,10 +45,14 @@ import com.google.common.collect.ImmutableMap;
   // Format specific parameter names.
   // Format specific parameter names.
   private static final String PARAMETER_PROFILE_LEVEL_ID = "profile-level-id";
   private static final String PARAMETER_PROFILE_LEVEL_ID = "profile-level-id";
   private static final String PARAMETER_SPROP_PARAMS = "sprop-parameter-sets";
   private static final String PARAMETER_SPROP_PARAMS = "sprop-parameter-sets";
+  private static final String PARAMETER_CONFIG = "config";
+
   /** Prefix for the RFC6381 codecs string for AAC formats. */
   /** Prefix for the RFC6381 codecs string for AAC formats. */
   private static final String AAC_CODECS_PREFIX = "mp4a.40.";
   private static final String AAC_CODECS_PREFIX = "mp4a.40.";
   /** Prefix for the RFC6381 codecs string for AVC formats. */
   /** Prefix for the RFC6381 codecs string for AVC formats. */
   private static final String H264_CODECS_PREFIX = "avc1.";
   private static final String H264_CODECS_PREFIX = "avc1.";
+  /** Prefix for the RFC6416 codecs string for MPEG4V-ES formats. */
+  private static final String MPEG4_CODECS_PREFIX = "mp4v";
 
 
   private static final String GENERIC_CONTROL_ATTR = "*";
   private static final String GENERIC_CONTROL_ATTR = "*";
 
 
@@ -116,6 +121,10 @@ import com.google.common.collect.ImmutableMap;
         checkArgument(!fmtpParameters.isEmpty());
         checkArgument(!fmtpParameters.isEmpty());
         processAacFmtpAttribute(formatBuilder, fmtpParameters, channelCount, clockRate);
         processAacFmtpAttribute(formatBuilder, fmtpParameters, channelCount, clockRate);
         break;
         break;
+      case MimeTypes.VIDEO_MP4V:
+        checkArgument(!fmtpParameters.isEmpty());
+        processMPEG4FmtpAttribute(formatBuilder, fmtpParameters);
+        break;
       case MimeTypes.VIDEO_H264:
       case MimeTypes.VIDEO_H264:
         checkArgument(!fmtpParameters.isEmpty());
         checkArgument(!fmtpParameters.isEmpty());
         processH264FmtpAttribute(formatBuilder, fmtpParameters);
         processH264FmtpAttribute(formatBuilder, fmtpParameters);
@@ -160,6 +169,24 @@ import com.google.common.collect.ImmutableMap;
             AacUtil.buildAacLcAudioSpecificConfig(sampleRate, channelCount)));
             AacUtil.buildAacLcAudioSpecificConfig(sampleRate, channelCount)));
   }
   }
 
 
+  private static void processMPEG4FmtpAttribute(
+      Format.Builder formatBuilder, ImmutableMap<String, String> fmtpAttributes) {
+    @Nullable String configInput = fmtpAttributes.get(PARAMETER_CONFIG);
+    if (configInput != null) {
+      byte[] csd = Util.getBytesFromHexString(configInput);
+      ImmutableList<byte[]> initializationData = ImmutableList.of(csd);
+      formatBuilder.setInitializationData(initializationData);
+      Pair<Integer, Integer> dimensions = CodecSpecificDataUtil.parseMpeg4VideoSpecificConfig(csd);
+      formatBuilder.setWidth(dimensions.first);
+      formatBuilder.setHeight(dimensions.second);
+    }
+    @Nullable String profileLevel = fmtpAttributes.get(PARAMETER_PROFILE_LEVEL_ID);
+    if (profileLevel == null) {
+      profileLevel = "1"; // default
+    }
+    formatBuilder.setCodecs(MPEG4_CODECS_PREFIX + profileLevel);
+  }
+
   private static void processH264FmtpAttribute(
   private static void processH264FmtpAttribute(
       Format.Builder formatBuilder, ImmutableMap<String, String> fmtpAttributes) {
       Format.Builder formatBuilder, ImmutableMap<String, String> fmtpAttributes) {
     checkArgument(fmtpAttributes.containsKey(PARAMETER_SPROP_PARAMS));
     checkArgument(fmtpAttributes.containsKey(PARAMETER_SPROP_PARAMS));

libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java

@@ -38,6 +38,8 @@ import androidx.media3.exoplayer.rtsp.RtpPayloadFormat;
         return new RtpAacReader(payloadFormat);
         return new RtpAacReader(payloadFormat);
       case MimeTypes.VIDEO_H264:
       case MimeTypes.VIDEO_H264:
         return new RtpH264Reader(payloadFormat);
         return new RtpH264Reader(payloadFormat);
+      case MimeTypes.VIDEO_MP4V:
+        return new RtpMPEG4Reader(payloadFormat);
       default:
       default:
         // No supported reader, returning null.
         // No supported reader, returning null.
     }
     }

libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java

+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.exoplayer.rtsp.reader;
+
+import static androidx.media3.common.util.Assertions.checkStateNotNull;
+import static androidx.media3.common.util.Util.castNonNull;
+
+import androidx.media3.common.C;
+import androidx.media3.common.ParserException;
+import androidx.media3.common.util.Log;
+import androidx.media3.common.util.ParsableByteArray;
+import androidx.media3.common.util.Util;
+import androidx.media3.exoplayer.rtsp.RtpPayloadFormat;
+import androidx.media3.extractor.ExtractorOutput;
+import androidx.media3.extractor.TrackOutput;
+import com.google.common.primitives.Bytes;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+
+/**
+ * Parses an H265 byte stream carried on RTP packets, and extracts H265 Access Units. Refer to
+ * RFC6416 for more details.
+ */
+/* package */ final class RtpMPEG4Reader implements RtpPayloadReader {
+  private static final String TAG = "RtpMPEG4Reader";
+
+  private static final long MEDIA_CLOCK_FREQUENCY = 90_000;
+
+  /**
+   * VOP unit type.
+   */
+  private static final int I_VOP = 0;
+
+  private final RtpPayloadFormat payloadFormat;
+
+  private @MonotonicNonNull TrackOutput trackOutput;
+  @C.BufferFlags private int bufferFlags;
+
+  private long firstReceivedTimestamp;
+
+  private int previousSequenceNumber;
+
+  private long startTimeOffsetUs;
+
+  private int sampleLength;
+
+  File output = null;
+
+  FileOutputStream outputStream = null;
+
+  /** Creates an instance. */
+  public RtpMPEG4Reader(RtpPayloadFormat payloadFormat) {
+    this.payloadFormat = payloadFormat;
+    firstReceivedTimestamp = C.TIME_UNSET;
+    previousSequenceNumber = C.INDEX_UNSET;
+    sampleLength = 0;
+    try {
+      output = new File("/data/local/tmp/" + "mpeg4v_es.out");
+      outputStream = new FileOutputStream(output);
+    } catch (IOException e) {
+      //do nothing;
+    }
+  }
+
+  private static long toSampleUs(
+      long startTimeOffsetUs, long rtpTimestamp, long firstReceivedRtpTimestamp) {
+    return startTimeOffsetUs
+        + Util.scaleLargeTimestamp(
+            (rtpTimestamp - firstReceivedRtpTimestamp),
+            /* multiplier= */ C.MICROS_PER_SECOND,
+            /* divisor= */ MEDIA_CLOCK_FREQUENCY);
+  }
+
+  @Override
+  public void createTracks(ExtractorOutput extractorOutput, int trackId) {
+    trackOutput = extractorOutput.track(trackId, C.TRACK_TYPE_VIDEO);
+    castNonNull(trackOutput).format(payloadFormat.format);
+  }
+
+  @Override
+  public void onReceivingFirstPacket(long timestamp, int sequenceNumber) {
+    Log.i(TAG, "RtpMPEG4Reader onReceivingFirstPacket");
+  }
+
+  @Override
+  public void consume(ParsableByteArray data, long timestamp, int sequenceNumber, boolean rtpMarker)
+      throws ParserException {
+    if (previousSequenceNumber != C.INDEX_UNSET && sequenceNumber != (previousSequenceNumber + 1)) {
+      Log.e(TAG, "Packet loss");
+    }
+    checkStateNotNull(trackOutput);
+
+    int limit = data.bytesLeft();
+    trackOutput.sampleData(data, limit);
+    sampleLength += limit;
+    parseVopType(data);
+
+    // Write the video sample
+    if (outputStream != null) {
+      try {
+        outputStream.write(data.getData());
+      } catch (IOException e) {
+        e.printStackTrace();
+      }
+    }
+
+    // Marker (M) bit: The marker bit is set to 1 to indicate the last RTP
+    // packet(or only RTP packet) of a VOP. When multiple VOPs are carried
+    // in the same RTP packet, the marker bit is set to 1.
+    if (rtpMarker) {
+        if (firstReceivedTimestamp == C.TIME_UNSET) {
+          firstReceivedTimestamp = timestamp;
+        }
+
+        long timeUs = toSampleUs(startTimeOffsetUs, timestamp, firstReceivedTimestamp);
+        trackOutput.sampleMetadata(timeUs, bufferFlags, sampleLength, 0, null);
+        sampleLength = 0;
+      }
+
+    previousSequenceNumber = sequenceNumber;
+  }
+
+  /**
+   * Parses VOP Coding type
+   *
+   * Sets {@link #bufferFlags} according to the VOP Coding type.
+   */
+  private void parseVopType(ParsableByteArray data) {
+    // search for VOP_START_CODE (00 00 01 B6)
+    byte[] inputData = data.getData();
+    byte[] startCode = {0x0, 0x0, 0x01, (byte) 0xB6};
+    int vopStartCodePos = Bytes.indexOf(inputData, startCode);
+    if (vopStartCodePos != -1) {
+      data.setPosition(vopStartCodePos + 4);
+      int vopType = data.peekUnsignedByte() >> 6;
+      bufferFlags = getBufferFlagsFromVopType(vopType);
+    }
+  }
+
+  @C.BufferFlags
+  private static int getBufferFlagsFromVopType(int vopType) {
+    return vopType == I_VOP ? C.BUFFER_FLAG_KEY_FRAME : 0;
+  }
+
+  @Override
+  public void seek(long nextRtpTimestamp, long timeUs) {
+    firstReceivedTimestamp = nextRtpTimestamp;
+    startTimeOffsetUs = timeUs;
+    sampleLength = 0;
+  }
+}