Skip to content

SDK / exoplayer

Go to a project
Commit e36ea3b7 by Ian Baker
Options

Merge pull request #36 from ittiam-systems:rtp-h265 

PiperOrigin-RevId: 429566102
parents c6805175 ca1c1c26
Inline Side-by-side
Showing with 346 additions and 19 deletions
RELEASENOTES.md
......@@ -152,6 +152,8 @@
* Remove deprecated call to `onStop(/* reset= */ true)` and provide an
opt-out flag for apps that don't want to clear the playlist on stop.
* RTSP:
* Add RTP reader for HEVC
([#36](https://github.com/androidx/media/pull/36))
* Provide a client API to override the `SocketFactory` used for any server
connection ([#9606](https://github.com/google/ExoPlayer/pull/9606)).
* Prefers DIGEST authentication method over BASIC if both are present
......
library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/RtpPayloadFormat.java
......@@ -37,12 +37,14 @@ public final class RtpPayloadFormat {
private static final String RTP_MEDIA_AC3 = "AC3";
private static final String RTP_MEDIA_MPEG4_GENERIC = "MPEG4-GENERIC";
private static final String RTP_MEDIA_H264 = "H264";
private static final String RTP_MEDIA_H265 = "H265";
/** Returns whether the format of a {@link MediaDescription} is supported. */
public static boolean isFormatSupported(MediaDescription mediaDescription) {
switch (Ascii.toUpperCase(mediaDescription.rtpMapAttribute.mediaEncoding)) {
case RTP_MEDIA_AC3:
case RTP_MEDIA_H264:
case RTP_MEDIA_H265:
case RTP_MEDIA_MPEG4_GENERIC:
return true;
default:
......@@ -63,6 +65,8 @@ public final class RtpPayloadFormat {
return MimeTypes.AUDIO_AC3;
case RTP_MEDIA_H264:
return MimeTypes.VIDEO_H264;
case RTP_MEDIA_H265:
return MimeTypes.VIDEO_H265;
case RTP_MEDIA_MPEG4_GENERIC:
return MimeTypes.AUDIO_AAC;
default:
......
library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/RtspMediaTrack.java
......@@ -42,6 +42,11 @@ import com.google.common.collect.ImmutableMap;
// Format specific parameter names.
private static final String PARAMETER_PROFILE_LEVEL_ID = "profile-level-id";
private static final String PARAMETER_SPROP_PARAMS = "sprop-parameter-sets";
private static final String PARAMETER_H265_SPROP_SPS = "sprop-sps";
private static final String PARAMETER_H265_SPROP_PPS = "sprop-pps";
private static final String PARAMETER_H265_SPROP_VPS = "sprop-vps";
private static final String PARAMETER_H265_SPROP_MAX_DON_DIFF = "sprop-max-don-diff";
/** Prefix for the RFC6381 codecs string for AAC formats. */
private static final String AAC_CODECS_PREFIX = "mp4a.40.";
/** Prefix for the RFC6381 codecs string for AVC formats. */
......@@ -118,6 +123,10 @@ import com.google.common.collect.ImmutableMap;
checkArgument(!fmtpParameters.isEmpty());
processH264FmtpAttribute(formatBuilder, fmtpParameters);
break;
case MimeTypes.VIDEO_H265:
checkArgument(!fmtpParameters.isEmpty());
processH265FmtpAttribute(formatBuilder, fmtpParameters);
break;
case MimeTypes.AUDIO_AC3:
// AC3 does not require a FMTP attribute. Fall through.
default:
......@@ -158,6 +167,26 @@ import com.google.common.collect.ImmutableMap;
AacUtil.buildAacLcAudioSpecificConfig(sampleRate, channelCount)));
}
/** Returns H264/H265 initialization data from the RTP parameter set. */
private static byte[] getInitializationDataFromParameterSet(String parameterSet) {
byte[] decodedParameterNalData = Base64.decode(parameterSet, Base64.DEFAULT);
byte[] decodedParameterNalUnit =
new byte[decodedParameterNalData.length + NAL_START_CODE.length];
System.arraycopy(
NAL_START_CODE,
/* srcPos= */ 0,
decodedParameterNalUnit,
/* destPos= */ 0,
NAL_START_CODE.length);
System.arraycopy(
decodedParameterNalData,
/* srcPos= */ 0,
decodedParameterNalUnit,
/* destPos= */ NAL_START_CODE.length,
decodedParameterNalData.length);
return decodedParameterNalUnit;
}
private static void processH264FmtpAttribute(
Format.Builder formatBuilder, ImmutableMap<String, String> fmtpAttributes) {
checkArgument(fmtpAttributes.containsKey(PARAMETER_SPROP_PARAMS));
......@@ -166,8 +195,8 @@ import com.google.common.collect.ImmutableMap;
checkArgument(parameterSets.length == 2);
ImmutableList<byte[]> initializationData =
ImmutableList.of(
getH264InitializationDataFromParameterSet(parameterSets[0]),
getH264InitializationDataFromParameterSet(parameterSets[1]));
getInitializationDataFromParameterSet(parameterSets[0]),
getInitializationDataFromParameterSet(parameterSets[1]));
formatBuilder.setInitializationData(initializationData);
// Process SPS (Sequence Parameter Set).
......@@ -189,23 +218,44 @@ import com.google.common.collect.ImmutableMap;
}
}
private static byte[] getH264InitializationDataFromParameterSet(String parameterSet) {
byte[] decodedParameterNalData = Base64.decode(parameterSet, Base64.DEFAULT);
byte[] decodedParameterNalUnit =
new byte[decodedParameterNalData.length + NAL_START_CODE.length];
System.arraycopy(
NAL_START_CODE,
/* srcPos= */ 0,
decodedParameterNalUnit,
/* destPos= */ 0,
NAL_START_CODE.length);
System.arraycopy(
decodedParameterNalData,
/* srcPos= */ 0,
decodedParameterNalUnit,
/* destPos= */ NAL_START_CODE.length,
decodedParameterNalData.length);
return decodedParameterNalUnit;
private static void processH265FmtpAttribute(
Format.Builder formatBuilder, ImmutableMap<String, String> fmtpAttributes) {
if (fmtpAttributes.containsKey(PARAMETER_H265_SPROP_MAX_DON_DIFF)) {
int maxDonDiff =
Integer.parseInt(checkNotNull(fmtpAttributes.get(PARAMETER_H265_SPROP_MAX_DON_DIFF)));
checkArgument(
maxDonDiff == 0, "non-zero sprop-max-don-diff " + maxDonDiff + " is not supported");
}
checkArgument(fmtpAttributes.containsKey(PARAMETER_H265_SPROP_VPS));
String spropVPS = checkNotNull(fmtpAttributes.get(PARAMETER_H265_SPROP_VPS));
checkArgument(fmtpAttributes.containsKey(PARAMETER_H265_SPROP_SPS));
String spropSPS = checkNotNull(fmtpAttributes.get(PARAMETER_H265_SPROP_SPS));
checkArgument(fmtpAttributes.containsKey(PARAMETER_H265_SPROP_PPS));
String spropPPS = checkNotNull(fmtpAttributes.get(PARAMETER_H265_SPROP_PPS));
ImmutableList<byte[]> initializationData =
ImmutableList.of(
getInitializationDataFromParameterSet(spropVPS),
getInitializationDataFromParameterSet(spropSPS),
getInitializationDataFromParameterSet(spropPPS));
formatBuilder.setInitializationData(initializationData);
// Process the SPS (Sequence Parameter Set).
byte[] spsNalDataWithStartCode = initializationData.get(1);
NalUnitUtil.H265SpsData spsData =
NalUnitUtil.parseH265SpsNalUnit(
spsNalDataWithStartCode, NAL_START_CODE.length, spsNalDataWithStartCode.length);
formatBuilder.setPixelWidthHeightRatio(spsData.pixelWidthHeightRatio);
formatBuilder.setHeight(spsData.height).setWidth(spsData.width);
formatBuilder.setCodecs(
CodecSpecificDataUtil.buildHevcCodecString(
spsData.generalProfileSpace,
spsData.generalTierFlag,
spsData.generalProfileIdc,
spsData.generalProfileCompatibilityFlags,
spsData.constraintBytes,
spsData.generalLevelIdc));
}
/**
......
library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/reader/DefaultRtpPayloadReaderFactory.java
......@@ -36,6 +36,8 @@ import com.google.android.exoplayer2.util.MimeTypes;
return new RtpAacReader(payloadFormat);
case MimeTypes.VIDEO_H264:
return new RtpH264Reader(payloadFormat);
case MimeTypes.VIDEO_H265:
return new RtpH265Reader(payloadFormat);
default:
// No supported reader, returning null.
}
......
library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/reader/RtpH265Reader.java 0 → 100644
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.source.rtsp.reader;
import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.ParserException;
import com.google.android.exoplayer2.extractor.ExtractorOutput;
import com.google.android.exoplayer2.extractor.TrackOutput;
import com.google.android.exoplayer2.source.rtsp.RtpPacket;
import com.google.android.exoplayer2.source.rtsp.RtpPayloadFormat;
import com.google.android.exoplayer2.util.Log;
import com.google.android.exoplayer2.util.NalUnitUtil;
import com.google.android.exoplayer2.util.ParsableByteArray;
import com.google.android.exoplayer2.util.Util;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
/**
* Parses an H265 byte stream carried on RTP packets, and extracts H265 Access Units. Refer to
* RFC7798 for more details.
*/
/* package */ final class RtpH265Reader implements RtpPayloadReader {
private static final String TAG = "RtpH265Reader";
private static final long MEDIA_CLOCK_FREQUENCY = 90_000;
/** Offset of payload data within a FU payload. */
private static final int FU_PAYLOAD_OFFSET = 3;
/** Aggregation Packet. RFC7798 Section 4.4.2. */
private static final int RTP_PACKET_TYPE_AP = 48;
/** Fragmentation Unit. RFC7798 Section 4.4.3. */
private static final int RTP_PACKET_TYPE_FU = 49;
/** IDR NAL unit types. */
private static final int NAL_IDR_W_RADL = 19;
private static final int NAL_IDR_N_LP = 20;
/** Scratch for Fragmentation Unit RTP packets. */
private final ParsableByteArray fuScratchBuffer;
private final ParsableByteArray nalStartCodeArray;
private final RtpPayloadFormat payloadFormat;
private @MonotonicNonNull TrackOutput trackOutput;
private @C.BufferFlags int bufferFlags;
private long firstReceivedTimestamp;
private int previousSequenceNumber;
/** The combined size of a sample that is fragmented into multiple RTP packets. */
private int fragmentedSampleSizeBytes;
private long startTimeOffsetUs;
/** Creates an instance. */
public RtpH265Reader(RtpPayloadFormat payloadFormat) {
this.fuScratchBuffer = new ParsableByteArray();
this.nalStartCodeArray = new ParsableByteArray(NalUnitUtil.NAL_START_CODE);
this.payloadFormat = payloadFormat;
firstReceivedTimestamp = C.TIME_UNSET;
previousSequenceNumber = C.INDEX_UNSET;
}
@Override
public void createTracks(ExtractorOutput extractorOutput, int trackId) {
trackOutput = extractorOutput.track(trackId, C.TRACK_TYPE_VIDEO);
trackOutput.format(payloadFormat.format);
}
@Override
public void onReceivingFirstPacket(long timestamp, int sequenceNumber) {}
@Override
public void consume(ParsableByteArray data, long timestamp, int sequenceNumber, boolean rtpMarker)
throws ParserException {
if (data.getData().length == 0) {
throw ParserException.createForMalformedManifest("Empty RTP data packet.", /* cause= */ null);
}
// NAL Unit Header.type (RFC7798 Section 1.1.4).
int payloadType = (data.getData()[0] >> 1) & 0x3F;
checkStateNotNull(trackOutput);
if (payloadType >= 0 && payloadType < RTP_PACKET_TYPE_AP) {
processSingleNalUnitPacket(data);
} else if (payloadType == RTP_PACKET_TYPE_AP) {
// TODO: Support AggregationPacket mode.
throw new UnsupportedOperationException("need to implement processAggregationPacket");
} else if (payloadType == RTP_PACKET_TYPE_FU) {
processFragmentationUnitPacket(data, sequenceNumber);
} else {
throw ParserException.createForMalformedManifest(
String.format("RTP H265 payload type [%d] not supported.", payloadType),
/* cause= */ null);
}
if (rtpMarker) {
if (firstReceivedTimestamp == C.TIME_UNSET) {
firstReceivedTimestamp = timestamp;
}
long timeUs = toSampleUs(startTimeOffsetUs, timestamp, firstReceivedTimestamp);
trackOutput.sampleMetadata(
timeUs, bufferFlags, fragmentedSampleSizeBytes, /* offset= */ 0, /* cryptoData= */ null);
fragmentedSampleSizeBytes = 0;
}
previousSequenceNumber = sequenceNumber;
}
@Override
public void seek(long nextRtpTimestamp, long timeUs) {
firstReceivedTimestamp = nextRtpTimestamp;
fragmentedSampleSizeBytes = 0;
startTimeOffsetUs = timeUs;
}
// Internal methods.
/**
* Processes Single NAL Unit packet (RFC7798 Section 4.4.1).
*
* <p>Outputs the single NAL Unit (with start code prepended) to {@link #trackOutput}. Sets {@link
* #bufferFlags} and {@link #fragmentedSampleSizeBytes} accordingly.
*/
@RequiresNonNull("trackOutput")
private void processSingleNalUnitPacket(ParsableByteArray data) {
// The structure a single NAL unit packet.
// 0 1 2 3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | PayloadHdr | DONL (conditional) |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | |
// | NAL unit payload data |
// | |
// | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | :...OPTIONAL RTP padding |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
int numBytesInData = data.bytesLeft();
fragmentedSampleSizeBytes += writeStartCode();
trackOutput.sampleData(data, numBytesInData);
fragmentedSampleSizeBytes += numBytesInData;
int nalHeaderType = (data.getData()[0] >> 1) & 0x3F;
bufferFlags = getBufferFlagsFromNalType(nalHeaderType);
}
/**
* Processes Fragmentation Unit packet (RFC7798 Section 4.4.3).
*
* <p>This method will be invoked multiple times to receive a single frame that is broken down
* into a series of fragmentation units in multiple RTP packets.
*
* <p>Outputs the received fragmentation units (with start code prepended) to {@link
* #trackOutput}. Sets {@link #bufferFlags} and {@link #fragmentedSampleSizeBytes} accordingly.
*/
@RequiresNonNull("trackOutput")
private void processFragmentationUnitPacket(ParsableByteArray data, int packetSequenceNumber)
throws ParserException {
// The structure of an FU packet.
// 0 1 2 3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | PayloadHdr (Type=49) | FU header | DONL (cond) |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-|
// | DONL (cond) | |
// |-+-+-+-+-+-+-+-+ |
// | FU payload |
// | |
// | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | :...OPTIONAL RTP padding |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
//
// FU header.
// +---------------+
// |0|1|2|3|4|5|6|7|
// +-+-+-+-+-+-+-+-+
// |S|E| FuType |
// +---------------+
//
// Structure of the PayloadHdr and HEVC NAL unit header, RFC7798 Section 1.1.4.
// +---------------+---------------+
// |0|1|2|3|4|5|6|7|0|1|2|3|4|5|6|7|
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |F| Type | LayerId | TID |
// +-------------+-----------------+
if (data.getData().length < 3) {
throw ParserException.createForMalformedManifest("Malformed FU header.", /* cause= */ null);
}
int tid = (data.getData()[1] & 0x7);
int fuHeader = data.getData()[2];
int nalUnitType = fuHeader & 0x3F;
boolean isFirstFuPacket = (fuHeader & 0x80) > 0;
boolean isLastFuPacket = (fuHeader & 0x40) > 0;
if (isFirstFuPacket) {
// Prepends starter code.
fragmentedSampleSizeBytes += writeStartCode();
// Convert RTP header into HEVC NAL Unit header accoding to RFC7798 Section 1.1.4.
// RTP byte 0: ignored.
// RTP byte 1: repurposed as HEVC HALU byte 0, copy NALU type.
// RTP Byte 2: repurposed as HEVC HALU byte 1, layerId required to be zero, copying only tid.
// Set data position from byte 1 as byte 0 is ignored.
data.getData()[1] = (byte) ((nalUnitType << 1) & 0x7F);
data.getData()[2] = (byte) tid;
fuScratchBuffer.reset(data.getData());
fuScratchBuffer.setPosition(1);
} else {
// Check that this packet is in the sequence of the previous packet.
int expectedSequenceNumber = (previousSequenceNumber + 1) % RtpPacket.MAX_SEQUENCE_NUMBER;
if (packetSequenceNumber != expectedSequenceNumber) {
Log.w(
TAG,
Util.formatInvariant(
"Received RTP packet with unexpected sequence number. Expected: %d; received: %d."
+ " Dropping packet.",
expectedSequenceNumber, packetSequenceNumber));
return;
}
// Setting position to ignore payload and FU header.
fuScratchBuffer.reset(data.getData());
fuScratchBuffer.setPosition(FU_PAYLOAD_OFFSET);
}
int fragmentSize = fuScratchBuffer.bytesLeft();
trackOutput.sampleData(fuScratchBuffer, fragmentSize);
fragmentedSampleSizeBytes += fragmentSize;
if (isLastFuPacket) {
bufferFlags = getBufferFlagsFromNalType(nalUnitType);
}
}
private int writeStartCode() {
nalStartCodeArray.setPosition(/* position= */ 0);
int bytesWritten = nalStartCodeArray.bytesLeft();
checkNotNull(trackOutput).sampleData(nalStartCodeArray, bytesWritten);
return bytesWritten;
}
private static long toSampleUs(
long startTimeOffsetUs, long rtpTimestamp, long firstReceivedRtpTimestamp) {
return startTimeOffsetUs
+ Util.scaleLargeTimestamp(
(rtpTimestamp - firstReceivedRtpTimestamp),
/* multiplier= */ C.MICROS_PER_SECOND,
/* divisor= */ MEDIA_CLOCK_FREQUENCY);
}
private static @C.BufferFlags int getBufferFlagsFromNalType(int nalType) {
return (nalType == NAL_IDR_W_RADL || nalType == NAL_IDR_N_LP) ? C.BUFFER_FLAG_KEY_FRAME : 0;
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment