Skip to content

SDK / exoplayer

Go to a project
Commit 56c070a6 by huangdarwin Committed by Tofunmi Adigun-Hameed
Options

Transformer: Add codec support for Dolby Vision HDR video 

Allow use of H265/H264 codecs for Dolby Vision video.

Also, reflow ExoPlayer code to use this new utility class

PiperOrigin-RevId: 530619388
parent 3a70c6ca
Inline Side-by-side
Showing with 166 additions and 63 deletions
library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java
...@@ -393,20 +393,8 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media ...@@ -393,20 +393,8 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
return ImmutableList.of(codecInfo); return ImmutableList.of(codecInfo);
} }
} }
List<MediaCodecInfo> decoderInfos = return MediaCodecUtil.getDecoderInfosSoftMatch(
mediaCodecSelector.getDecoderInfos( mediaCodecSelector, format, requiresSecureDecoder, /* requiresTunnelingDecoder= */ false);
mimeType, requiresSecureDecoder, /* requiresTunnelingDecoder= */ false);
@Nullable String alternativeMimeType = MediaCodecUtil.getAlternativeCodecMimeType(format);
if (alternativeMimeType == null) {
return ImmutableList.copyOf(decoderInfos);
}
List<MediaCodecInfo> alternativeDecoderInfos =
mediaCodecSelector.getDecoderInfos(
alternativeMimeType, requiresSecureDecoder, /* requiresTunnelingDecoder= */ false);
return ImmutableList.<MediaCodecInfo>builder()
.addAll(decoderInfos)
.addAll(alternativeDecoderInfos)
.build();
} }
@Override @Override
......
library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecUtil.java
...@@ -43,6 +43,7 @@ import java.util.List; ...@@ -43,6 +43,7 @@ import java.util.List;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.checkerframework.checker.nullness.qual.EnsuresNonNull; import org.checkerframework.checker.nullness.qual.EnsuresNonNull;
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
/** A utility class for querying the available codecs. */ /** A utility class for querying the available codecs. */
@SuppressLint("InlinedApi") @SuppressLint("InlinedApi")
...@@ -188,6 +189,77 @@ public final class MediaCodecUtil { ...@@ -188,6 +189,77 @@ public final class MediaCodecUtil {
} }
/** /**
* Returns a list of decoders that can decode media in the specified format, in the priority order
* specified by the {@link MediaCodecSelector}.
*
* <p>Since the {@link MediaCodecSelector} only has access to {@link Format#sampleMimeType}, the
* list is not ordered to account for whether each decoder supports the details of the format
* (e.g., taking into account the format's profile, level, resolution and so on). {@link
* #getDecoderInfosSortedByFormatSupport} can be used to further sort the list into an order where
* decoders that fully support the format come first.
*
* <p>This list is more complete than {@link #getDecoderInfos}, as it also considers alternative
* MIME types that are a close match using {@link #getAlternativeCodecMimeType}.
*
* @param mediaCodecSelector The decoder selector.
* @param format The {@link Format} for which a decoder is required.
* @param requiresSecureDecoder Whether a secure decoder is required.
* @param requiresTunnelingDecoder Whether a tunneling decoder is required.
* @return A list of {@link MediaCodecInfo}s corresponding to decoders. May be empty.
* @throws DecoderQueryException Thrown if there was an error querying decoders.
*/
@RequiresNonNull("#2.sampleMimeType")
public static List<MediaCodecInfo> getDecoderInfosSoftMatch(
MediaCodecSelector mediaCodecSelector,
Format format,
boolean requiresSecureDecoder,
boolean requiresTunnelingDecoder)
throws DecoderQueryException {
List<MediaCodecInfo> decoderInfos =
mediaCodecSelector.getDecoderInfos(
format.sampleMimeType, requiresSecureDecoder, requiresTunnelingDecoder);
List<MediaCodecInfo> alternativeDecoderInfos =
getAlternativeDecoderInfos(
mediaCodecSelector, format, requiresSecureDecoder, requiresTunnelingDecoder);
return ImmutableList.<MediaCodecInfo>builder()
.addAll(decoderInfos)
.addAll(alternativeDecoderInfos)
.build();
}
/**
* Returns a list of decoders for {@linkplain #getAlternativeCodecMimeType alternative MIME types}
* that can decode samples of the provided {@link Format}, in the priority order specified by the
* {@link MediaCodecSelector}.
*
* <p>Since the {@link MediaCodecSelector} only has access to {@link Format#sampleMimeType}, the
* list is not ordered to account for whether each decoder supports the details of the format
* (e.g., taking into account the format's profile, level, resolution and so on). {@link
* #getDecoderInfosSortedByFormatSupport} can be used to further sort the list into an order where
* decoders that fully support the format come first.
*
* @param mediaCodecSelector The decoder selector.
* @param format The {@link Format} for which an alternative decoder is required.
* @param requiresSecureDecoder Whether a secure decoder is required.
* @param requiresTunnelingDecoder Whether a tunneling decoder is required.
* @return A list of {@link MediaCodecInfo}s corresponding to alternative decoders. May be empty.
* @throws DecoderQueryException Thrown if there was an error querying decoders.
*/
public static List<MediaCodecInfo> getAlternativeDecoderInfos(
MediaCodecSelector mediaCodecSelector,
Format format,
boolean requiresSecureDecoder,
boolean requiresTunnelingDecoder)
throws DecoderQueryException {
@Nullable String alternativeMimeType = getAlternativeCodecMimeType(format);
if (alternativeMimeType == null) {
return ImmutableList.of();
}
return mediaCodecSelector.getDecoderInfos(
alternativeMimeType, requiresSecureDecoder, requiresTunnelingDecoder);
}
/**
* Returns a copy of the provided decoder list sorted such that decoders with functional format * Returns a copy of the provided decoder list sorted such that decoders with functional format
* support are listed first. The returned list is modifiable for convenience. * support are listed first. The returned list is modifiable for convenience.
*/ */
...@@ -280,8 +352,7 @@ public final class MediaCodecUtil { ...@@ -280,8 +352,7 @@ public final class MediaCodecUtil {
// be done for profile CodecProfileLevel.DolbyVisionProfileDvheStn and profile // be done for profile CodecProfileLevel.DolbyVisionProfileDvheStn and profile
// CodecProfileLevel.DolbyVisionProfileDvheDtb because the first one is not backward // CodecProfileLevel.DolbyVisionProfileDvheDtb because the first one is not backward
// compatible and the second one is deprecated and is not always backward compatible. // compatible and the second one is deprecated and is not always backward compatible.
@Nullable @Nullable Pair<Integer, Integer> codecProfileAndLevel = getCodecProfileAndLevel(format);
Pair<Integer, Integer> codecProfileAndLevel = MediaCodecUtil.getCodecProfileAndLevel(format);
if (codecProfileAndLevel != null) { if (codecProfileAndLevel != null) {
int profile = codecProfileAndLevel.first; int profile = codecProfileAndLevel.first;
if (profile == CodecProfileLevel.DolbyVisionProfileDvheDtr if (profile == CodecProfileLevel.DolbyVisionProfileDvheDtr
......
library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java
...@@ -495,30 +495,21 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { ...@@ -495,30 +495,21 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
boolean requiresSecureDecoder, boolean requiresSecureDecoder,
boolean requiresTunnelingDecoder) boolean requiresTunnelingDecoder)
throws DecoderQueryException { throws DecoderQueryException {
@Nullable String mimeType = format.sampleMimeType; if (format.sampleMimeType == null) {
if (mimeType == null) {
return ImmutableList.of(); return ImmutableList.of();
} }
List<MediaCodecInfo> decoderInfos =
mediaCodecSelector.getDecoderInfos(
mimeType, requiresSecureDecoder, requiresTunnelingDecoder);
@Nullable String alternativeMimeType = MediaCodecUtil.getAlternativeCodecMimeType(format);
if (alternativeMimeType == null) {
return ImmutableList.copyOf(decoderInfos);
}
List<MediaCodecInfo> alternativeDecoderInfos =
mediaCodecSelector.getDecoderInfos(
alternativeMimeType, requiresSecureDecoder, requiresTunnelingDecoder);
if (Util.SDK_INT >= 26 if (Util.SDK_INT >= 26
&& MimeTypes.VIDEO_DOLBY_VISION.equals(format.sampleMimeType) && MimeTypes.VIDEO_DOLBY_VISION.equals(format.sampleMimeType)
&& !alternativeDecoderInfos.isEmpty()
&& !Api26.doesDisplaySupportDolbyVision(context)) { && !Api26.doesDisplaySupportDolbyVision(context)) {
return ImmutableList.copyOf(alternativeDecoderInfos); List<MediaCodecInfo> alternativeDecoderInfos =
MediaCodecUtil.getAlternativeDecoderInfos(
mediaCodecSelector, format, requiresSecureDecoder, requiresTunnelingDecoder);
if (!alternativeDecoderInfos.isEmpty()) {
return alternativeDecoderInfos;
}
} }
return ImmutableList.<MediaCodecInfo>builder() return MediaCodecUtil.getDecoderInfosSoftMatch(
.addAll(decoderInfos) mediaCodecSelector, format, requiresSecureDecoder, requiresTunnelingDecoder);
.addAll(alternativeDecoderInfos)
.build();
} }
@RequiresApi(26) @RequiresApi(26)
......
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/AndroidTestUtil.java
...@@ -148,6 +148,8 @@ public final class AndroidTestUtil { ...@@ -148,6 +148,8 @@ public final class AndroidTestUtil {
.setCodecs("hvc1.2.4.L153") .setCodecs("hvc1.2.4.L153")
.build(); .build();
public static final String MP4_ASSET_DOLBY_VISION_HDR = "asset:///media/mp4/dolbyVision-hdr.MOV";
public static final String MP4_ASSET_4K60_PORTRAIT_URI_STRING = public static final String MP4_ASSET_4K60_PORTRAIT_URI_STRING =
"asset:///media/mp4/portrait_4k60.mp4"; "asset:///media/mp4/portrait_4k60.mp4";
public static final Format MP4_ASSET_4K60_PORTRAIT_FORMAT = public static final Format MP4_ASSET_4K60_PORTRAIT_FORMAT =
......
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/HdrEditingTest.java
...@@ -17,6 +17,7 @@ package com.google.android.exoplayer2.transformer.mh; ...@@ -17,6 +17,7 @@ package com.google.android.exoplayer2.transformer.mh;
import static com.google.android.exoplayer2.transformer.AndroidTestUtil.MP4_ASSET_1080P_5_SECOND_HLG10; import static com.google.android.exoplayer2.transformer.AndroidTestUtil.MP4_ASSET_1080P_5_SECOND_HLG10;
import static com.google.android.exoplayer2.transformer.AndroidTestUtil.MP4_ASSET_720P_4_SECOND_HDR10; import static com.google.android.exoplayer2.transformer.AndroidTestUtil.MP4_ASSET_720P_4_SECOND_HDR10;
import static com.google.android.exoplayer2.transformer.AndroidTestUtil.MP4_ASSET_DOLBY_VISION_HDR;
import static com.google.android.exoplayer2.transformer.AndroidTestUtil.recordTestSkipped; import static com.google.android.exoplayer2.transformer.AndroidTestUtil.recordTestSkipped;
import static com.google.android.exoplayer2.transformer.mh.FileUtil.maybeAssertFileHasColorTransfer; import static com.google.android.exoplayer2.transformer.mh.FileUtil.maybeAssertFileHasColorTransfer;
import static com.google.android.exoplayer2.util.Assertions.checkNotNull; import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
...@@ -167,6 +168,33 @@ public class HdrEditingTest { ...@@ -167,6 +168,33 @@ public class HdrEditingTest {
} }
@Test @Test
public void exportAndTranscode_dolbyVisionFile_whenHdrEditingIsSupported_exports()
throws Exception {
String testId = "exportAndTranscode_dolbyVisionFile_whenHdrEditingIsSupported_exports";
Context context = ApplicationProvider.getApplicationContext();
// This dolby vision file has a ColorInfo identical to HLG10.
if (!deviceSupportsHdrEditing(VIDEO_H265, HLG10_DEFAULT_COLOR_INFO)) {
recordTestSkipped(context, testId, /* reason= */ "Device lacks HLG10 editing support.");
return;
}
Transformer transformer = new Transformer.Builder(context).build();
MediaItem mediaItem = MediaItem.fromUri(Uri.parse(MP4_ASSET_DOLBY_VISION_HDR));
ImmutableList<Effect> videoEffects =
ImmutableList.of(
new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build());
Effects effects = new Effects(/* audioProcessors= */ ImmutableList.of(), videoEffects);
EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(mediaItem).setEffects(effects).build();
ExportTestResult exportTestResult =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, editedMediaItem);
maybeAssertFileHasColorTransfer(exportTestResult.filePath, C.COLOR_TRANSFER_HLG);
}
@Test
public void exportAndTranscode_hdr10File_whenHdrEditingUnsupported_toneMapsOrThrows() public void exportAndTranscode_hdr10File_whenHdrEditingUnsupported_toneMapsOrThrows()
throws Exception { throws Exception {
String testId = "exportAndTranscode_hdr10File_whenHdrEditingUnsupported_toneMapsOrThrows"; String testId = "exportAndTranscode_hdr10File_whenHdrEditingUnsupported_toneMapsOrThrows";
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultDecoderFactory.java
...@@ -29,6 +29,7 @@ import android.view.Surface; ...@@ -29,6 +29,7 @@ import android.view.Surface;
import androidx.annotation.Nullable; import androidx.annotation.Nullable;
import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.mediacodec.MediaCodecInfo; import com.google.android.exoplayer2.mediacodec.MediaCodecInfo;
import com.google.android.exoplayer2.mediacodec.MediaCodecSelector;
import com.google.android.exoplayer2.mediacodec.MediaCodecUtil; import com.google.android.exoplayer2.mediacodec.MediaCodecUtil;
import com.google.android.exoplayer2.util.Log; import com.google.android.exoplayer2.util.Log;
import com.google.android.exoplayer2.util.MediaFormatUtil; import com.google.android.exoplayer2.util.MediaFormatUtil;
...@@ -145,10 +146,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -145,10 +146,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private static String getMediaCodecNameForDecoding(Format format) private static String getMediaCodecNameForDecoding(Format format)
throws MediaCodecUtil.DecoderQueryException, ExportException { throws MediaCodecUtil.DecoderQueryException, ExportException {
checkNotNull(format.sampleMimeType);
List<MediaCodecInfo> decoderInfos = List<MediaCodecInfo> decoderInfos =
MediaCodecUtil.getDecoderInfosSortedByFormatSupport( MediaCodecUtil.getDecoderInfosSortedByFormatSupport(
MediaCodecUtil.getDecoderInfos( MediaCodecUtil.getDecoderInfosSoftMatch(
checkNotNull(format.sampleMimeType), /* secure= */ false, /* tunneling= */ false), MediaCodecSelector.DEFAULT,
format,
/* requiresSecureDecoder= */ false,
/* requiresTunnelingDecoder= */ false),
format); format);
if (decoderInfos.isEmpty()) { if (decoderInfos.isEmpty()) {
throw createExportException(format, /* reason= */ "No decoders for format"); throw createExportException(format, /* reason= */ "No decoders for format");
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java
...@@ -31,6 +31,8 @@ import static com.google.android.exoplayer2.video.ColorInfo.isTransferHdr; ...@@ -31,6 +31,8 @@ import static com.google.android.exoplayer2.video.ColorInfo.isTransferHdr;
import android.content.Context; import android.content.Context;
import android.graphics.Bitmap; import android.graphics.Bitmap;
import android.media.MediaCodec; import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.util.Pair;
import android.view.Surface; import android.view.Surface;
import androidx.annotation.Nullable; import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting; import androidx.annotation.VisibleForTesting;
...@@ -39,6 +41,7 @@ import com.google.android.exoplayer2.Format; ...@@ -39,6 +41,7 @@ import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.decoder.DecoderInputBuffer; import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
import com.google.android.exoplayer2.effect.DebugTraceUtil; import com.google.android.exoplayer2.effect.DebugTraceUtil;
import com.google.android.exoplayer2.effect.Presentation; import com.google.android.exoplayer2.effect.Presentation;
import com.google.android.exoplayer2.mediacodec.MediaCodecUtil;
import com.google.android.exoplayer2.util.Consumer; import com.google.android.exoplayer2.util.Consumer;
import com.google.android.exoplayer2.util.DebugViewProvider; import com.google.android.exoplayer2.util.DebugViewProvider;
import com.google.android.exoplayer2.util.Effect; import com.google.android.exoplayer2.util.Effect;
...@@ -350,8 +353,16 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -350,8 +353,16 @@ import org.checkerframework.dataflow.qual.Pure;
this.muxerSupportedMimeTypes = muxerSupportedMimeTypes; this.muxerSupportedMimeTypes = muxerSupportedMimeTypes;
this.transformationRequest = transformationRequest; this.transformationRequest = transformationRequest;
this.fallbackListener = fallbackListener; this.fallbackListener = fallbackListener;
String inputSampleMimeType = checkNotNull(inputFormat.sampleMimeType); Pair<String, Integer> outputMimeTypeAndHdrModeAfterFallback =
getRequestedOutputMimeTypeAndHdrModeAfterFallback(inputFormat, transformationRequest);
requestedOutputMimeType = outputMimeTypeAndHdrModeAfterFallback.first;
hdrModeAfterFallback = outputMimeTypeAndHdrModeAfterFallback.second;
}
private static Pair<String, Integer> getRequestedOutputMimeTypeAndHdrModeAfterFallback(
Format inputFormat, TransformationRequest transformationRequest) {
String inputSampleMimeType = checkNotNull(inputFormat.sampleMimeType);
String requestedOutputMimeType;
if (transformationRequest.videoMimeType != null) { if (transformationRequest.videoMimeType != null) {
requestedOutputMimeType = transformationRequest.videoMimeType; requestedOutputMimeType = transformationRequest.videoMimeType;
} else if (MimeTypes.isImage(inputSampleMimeType)) { } else if (MimeTypes.isImage(inputSampleMimeType)) {
...@@ -362,33 +373,25 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -362,33 +373,25 @@ import org.checkerframework.dataflow.qual.Pure;
// HdrMode fallback is only supported from HDR_MODE_KEEP_HDR to // HdrMode fallback is only supported from HDR_MODE_KEEP_HDR to
// HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC. // HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC.
boolean fallbackToMediaCodec = @TransformationRequest.HdrMode int hdrMode = transformationRequest.hdrMode;
isTransferHdr(inputFormat.colorInfo) if (hdrMode == HDR_MODE_KEEP_HDR && isTransferHdr(inputFormat.colorInfo)) {
&& transformationRequest.hdrMode == HDR_MODE_KEEP_HDR ImmutableList<MediaCodecInfo> hdrEncoders =
&& getSupportedEncodersForHdrEditing(requestedOutputMimeType, inputFormat.colorInfo) getSupportedEncodersForHdrEditing(requestedOutputMimeType, inputFormat.colorInfo);
.isEmpty(); if (hdrEncoders.isEmpty()) {
hdrModeAfterFallback = @Nullable
fallbackToMediaCodec String alternativeMimeType = MediaCodecUtil.getAlternativeCodecMimeType(inputFormat);
? HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC if (alternativeMimeType != null) {
: transformationRequest.hdrMode; requestedOutputMimeType = alternativeMimeType;
} hdrEncoders =
getSupportedEncodersForHdrEditing(alternativeMimeType, inputFormat.colorInfo);
/** Returns the {@link ColorInfo} expected from the input surface. */ }
public ColorInfo getSupportedInputColor() { }
boolean isHdrEditingEnabled = if (hdrEncoders.isEmpty()) {
transformationRequest.hdrMode == HDR_MODE_KEEP_HDR hdrMode = HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC;
&& !getSupportedEncodersForHdrEditing(requestedOutputMimeType, inputFormat.colorInfo) }
.isEmpty();
boolean isInputToneMapped = !isHdrEditingEnabled && isTransferHdr(inputFormat.colorInfo);
if (isInputToneMapped) {
// When tone-mapping HDR to SDR is enabled, assume we get BT.709 to avoid having the encoder
// populate default color info, which depends on the resolution.
return ColorInfo.SDR_BT709_LIMITED;
}
if (SRGB_BT709_FULL.equals(inputFormat.colorInfo)) {
return ColorInfo.SDR_BT709_LIMITED;
} }
return checkNotNull(inputFormat.colorInfo);
return Pair.create(requestedOutputMimeType, hdrMode);
} }
public @TransformationRequest.HdrMode int getHdrModeAfterFallback() { public @TransformationRequest.HdrMode int getHdrModeAfterFallback() {
...@@ -458,6 +461,21 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -458,6 +461,21 @@ import org.checkerframework.dataflow.qual.Pure;
return encoderSurfaceInfo; return encoderSurfaceInfo;
} }
/** Returns the {@link ColorInfo} expected from the input surface. */
private ColorInfo getSupportedInputColor() {
boolean isInputToneMapped =
isTransferHdr(inputFormat.colorInfo) && hdrModeAfterFallback != HDR_MODE_KEEP_HDR;
if (isInputToneMapped) {
// When tone-mapping HDR to SDR is enabled, assume we get BT.709 to avoid having the encoder
// populate default color info, which depends on the resolution.
return ColorInfo.SDR_BT709_LIMITED;
}
if (SRGB_BT709_FULL.equals(inputFormat.colorInfo)) {
return ColorInfo.SDR_BT709_LIMITED;
}
return checkNotNull(inputFormat.colorInfo);
}
/** /**
* Creates a {@link TransformationRequest}, based on an original {@code TransformationRequest} * Creates a {@link TransformationRequest}, based on an original {@code TransformationRequest}
* and parameters specifying alterations to it that indicate device support. * and parameters specifying alterations to it that indicate device support.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment