Skip to content

SDK / exoplayer

Go to a project
Commit 0db07c67 by andrewlewis Committed by Rohit Singh
Options

Make minor fixes to HDR handling 

- Update profile selection logic to pick an HDR-compatible profile when doing HDR editing on H.264/AVC videos.
- Handle doing the capabilities check for all MIME types that support HDR (not just H.265/HEVC).
- Fix a bug where we would pass an HDR input color format to the encoder when using tone-mapping.
- Tweak how `EncoderWrapper` works so decisions at made at construction time.

Manually tested cases:
- Transformation of an SDR video.
- Transformation of an HDR video to AVC (which triggers fallback/tone-mapping on a device that doesn't support HDR editing for AVC).
- Transformation of an HDR video with HDR editing.

PiperOrigin-RevId: 461572973
parent 405be80f
Inline Side-by-side
Showing with 136 additions and 73 deletions
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultEncoderFactory.java
...@@ -273,7 +273,7 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { ...@@ -273,7 +273,7 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory {
} }
if (mimeType.equals(MimeTypes.VIDEO_H264)) { if (mimeType.equals(MimeTypes.VIDEO_H264)) {
adjustMediaFormatForH264EncoderSettings(mediaFormat, encoderInfo); adjustMediaFormatForH264EncoderSettings(format.colorInfo, encoderInfo, mediaFormat);
} }
MediaFormatUtil.maybeSetColorInfo(mediaFormat, encoderSupportedFormat.colorInfo); MediaFormatUtil.maybeSetColorInfo(mediaFormat, encoderSupportedFormat.colorInfo);
...@@ -523,12 +523,21 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { ...@@ -523,12 +523,21 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory {
* <p>The adjustment is applied in-place to {@code mediaFormat}. * <p>The adjustment is applied in-place to {@code mediaFormat}.
*/ */
private static void adjustMediaFormatForH264EncoderSettings( private static void adjustMediaFormatForH264EncoderSettings(
MediaFormat mediaFormat, MediaCodecInfo encoderInfo) { @Nullable ColorInfo colorInfo, MediaCodecInfo encoderInfo, MediaFormat mediaFormat) {
// TODO(b/210593256): Remove overriding profile/level (before API 29) after switching to in-app // TODO(b/210593256): Remove overriding profile/level (before API 29) after switching to in-app
// muxing. // muxing.
String mimeType = MimeTypes.VIDEO_H264; String mimeType = MimeTypes.VIDEO_H264;
if (Util.SDK_INT >= 29) { if (Util.SDK_INT >= 29) {
int expectedEncodingProfile = MediaCodecInfo.CodecProfileLevel.AVCProfileHigh; int expectedEncodingProfile = MediaCodecInfo.CodecProfileLevel.AVCProfileHigh;
if (colorInfo != null) {
int colorTransfer = colorInfo.colorTransfer;
ImmutableList<Integer> codecProfiles =
EncoderUtil.getCodecProfilesForHdrFormat(mimeType, colorTransfer);
if (!codecProfiles.isEmpty()) {
// Default to the most compatible profile, which is first in the list.
expectedEncodingProfile = codecProfiles.get(0);
}
}
int supportedEncodingLevel = int supportedEncodingLevel =
EncoderUtil.findHighestSupportedEncodingLevel( EncoderUtil.findHighestSupportedEncodingLevel(
encoderInfo, mimeType, expectedEncodingProfile); encoderInfo, mimeType, expectedEncodingProfile);
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/EncoderUtil.java
...@@ -31,10 +31,13 @@ import android.util.Size; ...@@ -31,10 +31,13 @@ import android.util.Size;
import androidx.annotation.DoNotInline; import androidx.annotation.DoNotInline;
import androidx.annotation.Nullable; import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi; import androidx.annotation.RequiresApi;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.C.ColorTransfer;
import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.util.MediaFormatUtil; import com.google.android.exoplayer2.util.MediaFormatUtil;
import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.MimeTypes;
import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.util.Util;
import com.google.android.exoplayer2.video.ColorInfo;
import com.google.common.base.Ascii; import com.google.common.base.Ascii;
import com.google.common.base.Supplier; import com.google.common.base.Supplier;
import com.google.common.base.Suppliers; import com.google.common.base.Suppliers;
...@@ -65,6 +68,83 @@ public final class EncoderUtil { ...@@ -65,6 +68,83 @@ public final class EncoderUtil {
return checkNotNull(MIME_TYPE_TO_ENCODERS.get()).keySet(); return checkNotNull(MIME_TYPE_TO_ENCODERS.get()).keySet();
} }
/**
* Returns the names of encoders that support HDR editing for the given format, or an empty list
* if the format is unknown or not supported for HDR encoding.
*/
public static ImmutableList<String> getSupportedEncoderNamesForHdrEditing(
String mimeType, @Nullable ColorInfo colorInfo) {
if (Util.SDK_INT < 31 || colorInfo == null) {
return ImmutableList.of();
}
@ColorTransfer int colorTransfer = colorInfo.colorTransfer;
ImmutableList<Integer> profiles = getCodecProfilesForHdrFormat(mimeType, colorTransfer);
ImmutableList.Builder<String> resultBuilder = ImmutableList.builder();
ImmutableList<MediaCodecInfo> mediaCodecInfos =
EncoderSelector.DEFAULT.selectEncoderInfos(mimeType);
for (int i = 0; i < mediaCodecInfos.size(); i++) {
MediaCodecInfo mediaCodecInfo = mediaCodecInfos.get(i);
if (mediaCodecInfo.isAlias()
|| !EncoderUtil.isFeatureSupported(
mediaCodecInfo, mimeType, MediaCodecInfo.CodecCapabilities.FEATURE_HdrEditing)) {
continue;
}
for (MediaCodecInfo.CodecProfileLevel codecProfileLevel :
mediaCodecInfo.getCapabilitiesForType(mimeType).profileLevels) {
if (profiles.contains(codecProfileLevel.profile)) {
resultBuilder.add(mediaCodecInfo.getName());
}
}
}
return resultBuilder.build();
}
/**
* Returns the {@linkplain MediaCodecInfo.CodecProfileLevel#profile profile} constants that can be
* used to encode the given HDR format, if supported by the device (this method does not check
* device capabilities). If multiple profiles are returned, they are ordered by expected level of
* compatibility, with the most widely compatible profile first.
*/
@SuppressWarnings("InlinedApi") // Safe use of inlined constants from newer API versions.
public static ImmutableList<Integer> getCodecProfilesForHdrFormat(
String mimeType, @ColorTransfer int colorTransfer) {
// TODO(b/239174610): Add a way to determine profiles for DV and HDR10+.
switch (mimeType) {
case MimeTypes.VIDEO_VP9:
if (colorTransfer == C.COLOR_TRANSFER_HLG || colorTransfer == C.COLOR_TRANSFER_ST2084) {
// Profiles support both HLG and PQ.
return ImmutableList.of(
MediaCodecInfo.CodecProfileLevel.VP9Profile2HDR,
MediaCodecInfo.CodecProfileLevel.VP9Profile3HDR);
}
break;
case MimeTypes.VIDEO_H264:
if (colorTransfer == C.COLOR_TRANSFER_HLG) {
return ImmutableList.of(MediaCodecInfo.CodecProfileLevel.AVCProfileHigh10);
}
break;
case MimeTypes.VIDEO_H265:
if (colorTransfer == C.COLOR_TRANSFER_HLG) {
return ImmutableList.of(MediaCodecInfo.CodecProfileLevel.HEVCProfileMain10);
} else if (colorTransfer == C.COLOR_TRANSFER_ST2084) {
return ImmutableList.of(MediaCodecInfo.CodecProfileLevel.HEVCProfileMain10HDR10);
}
break;
case MimeTypes.VIDEO_AV1:
if (colorTransfer == C.COLOR_TRANSFER_HLG) {
return ImmutableList.of(MediaCodecInfo.CodecProfileLevel.AV1ProfileMain10);
} else if (colorTransfer == C.COLOR_TRANSFER_ST2084) {
return ImmutableList.of(MediaCodecInfo.CodecProfileLevel.AV1ProfileMain10HDR10);
}
break;
default:
break;
}
// There are no profiles defined for the HDR format, or it's invalid.
return ImmutableList.of();
}
/** Returns whether the {@linkplain MediaCodecInfo encoder} supports the given resolution. */ /** Returns whether the {@linkplain MediaCodecInfo encoder} supports the given resolution. */
public static boolean isSizeSupported( public static boolean isSizeSupported(
MediaCodecInfo encoderInfo, String mimeType, int width, int height) { MediaCodecInfo encoderInfo, String mimeType, int width, int height) {
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java
...@@ -17,25 +17,20 @@ ...@@ -17,25 +17,20 @@
package com.google.android.exoplayer2.transformer; package com.google.android.exoplayer2.transformer;
import static com.google.android.exoplayer2.util.Assertions.checkNotNull; import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import static com.google.android.exoplayer2.util.Assertions.checkState;
import android.content.Context; import android.content.Context;
import android.media.MediaCodec; import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.view.Surface; import android.view.Surface;
import androidx.annotation.Nullable; import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting; import androidx.annotation.VisibleForTesting;
import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.decoder.DecoderInputBuffer; import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
import com.google.android.exoplayer2.util.Log;
import com.google.android.exoplayer2.util.MimeTypes;
import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.util.Util;
import com.google.android.exoplayer2.video.ColorInfo; import com.google.android.exoplayer2.video.ColorInfo;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashSet;
import java.util.List; import java.util.List;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.dataflow.qual.Pure; import org.checkerframework.dataflow.qual.Pure;
...@@ -105,17 +100,6 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -105,17 +100,6 @@ import org.checkerframework.dataflow.qual.Pure;
transformationRequest, transformationRequest,
fallbackListener); fallbackListener);
boolean enableRequestSdrToneMapping = transformationRequest.enableRequestSdrToneMapping;
// TODO(b/237674316): While HLG10 is correctly reported, HDR10 currently will be incorrectly
// processed as SDR, because the inputFormat.colorInfo reports the wrong value.
boolean useHdr =
transformationRequest.enableHdrEditing && ColorInfo.isHdr(inputFormat.colorInfo);
if (useHdr && !encoderWrapper.supportsHdr()) {
useHdr = false;
enableRequestSdrToneMapping = true;
encoderWrapper.signalFallbackToSdr();
}
try { try {
frameProcessor = frameProcessor =
GlEffectsFrameProcessor.create( GlEffectsFrameProcessor.create(
...@@ -153,7 +137,7 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -153,7 +137,7 @@ import org.checkerframework.dataflow.qual.Pure;
// HDR is only used if the MediaCodec encoder supports FEATURE_HdrEditing. This // HDR is only used if the MediaCodec encoder supports FEATURE_HdrEditing. This
// implies that the OpenGL EXT_YUV_target extension is supported and hence the // implies that the OpenGL EXT_YUV_target extension is supported and hence the
// GlEffectsFrameProcessor also supports HDR. // GlEffectsFrameProcessor also supports HDR.
useHdr); /* useHdr= */ encoderWrapper.isHdrEditingEnabled());
} catch (FrameProcessingException e) { } catch (FrameProcessingException e) {
throw TransformationException.createForFrameProcessingException( throw TransformationException.createForFrameProcessingException(
e, TransformationException.ERROR_CODE_GL_INIT_FAILED); e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
...@@ -161,9 +145,11 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -161,9 +145,11 @@ import org.checkerframework.dataflow.qual.Pure;
frameProcessor.setInputFrameInfo( frameProcessor.setInputFrameInfo(
new FrameInfo(decodedWidth, decodedHeight, inputFormat.pixelWidthHeightRatio)); new FrameInfo(decodedWidth, decodedHeight, inputFormat.pixelWidthHeightRatio));
boolean isToneMappingRequired =
ColorInfo.isHdr(inputFormat.colorInfo) && !encoderWrapper.isHdrEditingEnabled();
decoder = decoder =
decoderFactory.createForVideoDecoding( decoderFactory.createForVideoDecoding(
inputFormat, frameProcessor.getInputSurface(), enableRequestSdrToneMapping); inputFormat, frameProcessor.getInputSurface(), isToneMappingRequired);
// TODO(b/236316454): Check in the decoder output format whether tone-mapping was actually // TODO(b/236316454): Check in the decoder output format whether tone-mapping was actually
// applied and throw an exception if not. // applied and throw an exception if not.
maxPendingFrameCount = decoder.getMaxPendingFrameCount(); maxPendingFrameCount = decoder.getMaxPendingFrameCount();
...@@ -331,14 +317,14 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -331,14 +317,14 @@ import org.checkerframework.dataflow.qual.Pure;
private final List<String> allowedOutputMimeTypes; private final List<String> allowedOutputMimeTypes;
private final TransformationRequest transformationRequest; private final TransformationRequest transformationRequest;
private final FallbackListener fallbackListener; private final FallbackListener fallbackListener;
private final HashSet<String> hdrMediaCodecNames; private final String requestedOutputMimeType;
private final ImmutableList<String> supportedEncoderNamesForHdrEditing;
private @MonotonicNonNull SurfaceInfo encoderSurfaceInfo; private @MonotonicNonNull SurfaceInfo encoderSurfaceInfo;
private volatile @MonotonicNonNull Codec encoder; private volatile @MonotonicNonNull Codec encoder;
private volatile int outputRotationDegrees; private volatile int outputRotationDegrees;
private volatile boolean releaseEncoder; private volatile boolean releaseEncoder;
private boolean fallbackToSdr;
public EncoderWrapper( public EncoderWrapper(
Codec.EncoderFactory encoderFactory, Codec.EncoderFactory encoderFactory,
...@@ -346,14 +332,26 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -346,14 +332,26 @@ import org.checkerframework.dataflow.qual.Pure;
List<String> allowedOutputMimeTypes, List<String> allowedOutputMimeTypes,
TransformationRequest transformationRequest, TransformationRequest transformationRequest,
FallbackListener fallbackListener) { FallbackListener fallbackListener) {
this.encoderFactory = encoderFactory; this.encoderFactory = encoderFactory;
this.inputFormat = inputFormat; this.inputFormat = inputFormat;
this.allowedOutputMimeTypes = allowedOutputMimeTypes; this.allowedOutputMimeTypes = allowedOutputMimeTypes;
this.transformationRequest = transformationRequest; this.transformationRequest = transformationRequest;
this.fallbackListener = fallbackListener; this.fallbackListener = fallbackListener;
hdrMediaCodecNames = new HashSet<>(); requestedOutputMimeType =
transformationRequest.videoMimeType != null
? transformationRequest.videoMimeType
: checkNotNull(inputFormat.sampleMimeType);
supportedEncoderNamesForHdrEditing =
EncoderUtil.getSupportedEncoderNamesForHdrEditing(
requestedOutputMimeType, inputFormat.colorInfo);
}
/** Returns whether the wrapped encoder is expecting HDR input for the HDR editing use case. */
public boolean isHdrEditingEnabled() {
return transformationRequest.enableHdrEditing
&& !transformationRequest.enableRequestSdrToneMapping
&& !supportedEncoderNamesForHdrEditing.isEmpty();
} }
@Nullable @Nullable
...@@ -378,37 +376,39 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -378,37 +376,39 @@ import org.checkerframework.dataflow.qual.Pure;
outputRotationDegrees = 90; outputRotationDegrees = 90;
} }
boolean isInputToneMapped = ColorInfo.isHdr(inputFormat.colorInfo) && !isHdrEditingEnabled();
Format requestedEncoderFormat = Format requestedEncoderFormat =
new Format.Builder() new Format.Builder()
.setWidth(requestedWidth) .setWidth(requestedWidth)
.setHeight(requestedHeight) .setHeight(requestedHeight)
.setRotationDegrees(0) .setRotationDegrees(0)
.setFrameRate(inputFormat.frameRate) .setFrameRate(inputFormat.frameRate)
.setSampleMimeType( .setSampleMimeType(requestedOutputMimeType)
transformationRequest.videoMimeType != null .setColorInfo(isInputToneMapped ? null : inputFormat.colorInfo)
? transformationRequest.videoMimeType
: inputFormat.sampleMimeType)
.setColorInfo(fallbackToSdr ? null : inputFormat.colorInfo)
.build(); .build();
encoder = encoder =
encoderFactory.createForVideoEncoding(requestedEncoderFormat, allowedOutputMimeTypes); encoderFactory.createForVideoEncoding(requestedEncoderFormat, allowedOutputMimeTypes);
if (!hdrMediaCodecNames.isEmpty() && !hdrMediaCodecNames.contains(encoder.getName())) {
Log.d(
TAG,
"Selected encoder "
+ encoder.getName()
+ " does not report sufficient HDR capabilities");
}
Format encoderSupportedFormat = encoder.getConfigurationFormat(); Format encoderSupportedFormat = encoder.getConfigurationFormat();
if (isHdrEditingEnabled()) {
if (!requestedOutputMimeType.equals(encoderSupportedFormat.sampleMimeType)) {
throw createEncodingException(
new IllegalStateException("MIME type fallback unsupported with HDR editing"),
encoderSupportedFormat);
} else if (!supportedEncoderNamesForHdrEditing.contains(encoder.getName())) {
throw createEncodingException(
new IllegalStateException("Selected encoder doesn't support HDR editing"),
encoderSupportedFormat);
}
}
fallbackListener.onTransformationRequestFinalized( fallbackListener.onTransformationRequestFinalized(
createFallbackTransformationRequest( createFallbackTransformationRequest(
transformationRequest, transformationRequest,
/* hasOutputFormatRotation= */ flipOrientation, /* hasOutputFormatRotation= */ flipOrientation,
requestedEncoderFormat, requestedEncoderFormat,
encoderSupportedFormat, encoderSupportedFormat,
fallbackToSdr)); isInputToneMapped));
encoderSurfaceInfo = encoderSurfaceInfo =
new SurfaceInfo( new SurfaceInfo(
...@@ -468,41 +468,14 @@ import org.checkerframework.dataflow.qual.Pure; ...@@ -468,41 +468,14 @@ import org.checkerframework.dataflow.qual.Pure;
releaseEncoder = true; releaseEncoder = true;
} }
/** private TransformationException createEncodingException(Exception cause, Format format) {
* Checks whether at least one MediaCodec encoder on the device has sufficient capabilities to return TransformationException.createForCodec(
* encode HDR (only checks support for HLG at this time). cause,
*/ /* isVideo= */ true,
public boolean supportsHdr() { /* isDecoder= */ false,
if (Util.SDK_INT < 31) { format,
return false; checkNotNull(encoder).getName(),
} TransformationException.ERROR_CODE_ENCODING_FAILED);
// The only output MIME type that Transformer currently supports that can be used with HDR
// is H265/HEVC. So we assume that the EncoderFactory will pick this if HDR is requested.
String mimeType = MimeTypes.VIDEO_H265;
List<MediaCodecInfo> mediaCodecInfos = EncoderSelector.DEFAULT.selectEncoderInfos(mimeType);
for (int i = 0; i < mediaCodecInfos.size(); i++) {
MediaCodecInfo mediaCodecInfo = mediaCodecInfos.get(i);
if (EncoderUtil.isFeatureSupported(
mediaCodecInfo, mimeType, MediaCodecInfo.CodecCapabilities.FEATURE_HdrEditing)) {
for (MediaCodecInfo.CodecProfileLevel capabilities :
mediaCodecInfo.getCapabilitiesForType(MimeTypes.VIDEO_H265).profileLevels) {
// TODO(b/227624622): What profile to check depends on the HDR format. Once other
// formats besides HLG are supported, check the corresponding profiles here.
if (capabilities.profile == MediaCodecInfo.CodecProfileLevel.HEVCProfileMain10) {
return hdrMediaCodecNames.add(mediaCodecInfo.getCanonicalName());
}
}
}
}
return !hdrMediaCodecNames.isEmpty();
}
public void signalFallbackToSdr() {
checkState(encoder == null, "Fallback to SDR is only allowed before encoder initialization");
fallbackToSdr = true;
hdrMediaCodecNames.clear();
} }
} }
} }
library/transformer/src/test/java/com/google/android/exoplayer2/transformer/VideoEncoderWrapperTest.java
...@@ -27,6 +27,7 @@ import com.google.android.exoplayer2.Format; ...@@ -27,6 +27,7 @@ import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.MediaItem; import com.google.android.exoplayer2.MediaItem;
import com.google.android.exoplayer2.util.Clock; import com.google.android.exoplayer2.util.Clock;
import com.google.android.exoplayer2.util.ListenerSet; import com.google.android.exoplayer2.util.ListenerSet;
import com.google.android.exoplayer2.util.MimeTypes;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import java.util.List; import java.util.List;
import org.junit.Before; import org.junit.Before;
...@@ -47,7 +48,7 @@ public final class VideoEncoderWrapperTest { ...@@ -47,7 +48,7 @@ public final class VideoEncoderWrapperTest {
private final VideoTranscodingSamplePipeline.EncoderWrapper encoderWrapper = private final VideoTranscodingSamplePipeline.EncoderWrapper encoderWrapper =
new VideoTranscodingSamplePipeline.EncoderWrapper( new VideoTranscodingSamplePipeline.EncoderWrapper(
fakeEncoderFactory, fakeEncoderFactory,
/* inputFormat= */ new Format.Builder().build(), /* inputFormat= */ new Format.Builder().setSampleMimeType(MimeTypes.VIDEO_H265).build(),
/* allowedOutputMimeTypes= */ ImmutableList.of(), /* allowedOutputMimeTypes= */ ImmutableList.of(),
emptyTransformationRequest, emptyTransformationRequest,
fallbackListener); fallbackListener);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment