Skip to content

SDK / exoplayer

Go to a project
Commit d767be4c by claincly Committed by Ian Baker
Options

Add format fallback ranking. 

Introduce an interface EncoderSelector for developers to filter out unwanted
encoders.

PiperOrigin-RevId: 425611421
parent fc528ebe
Inline Side-by-side
Showing with 269 additions and 60 deletions
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Codec.java
...@@ -47,7 +47,7 @@ public final class Codec { ...@@ -47,7 +47,7 @@ public final class Codec {
public interface DecoderFactory { public interface DecoderFactory {
/** A default {@code DecoderFactory} implementation. */ /** A default {@code DecoderFactory} implementation. */
DecoderFactory DEFAULT = new DefaultCodecFactory(); DecoderFactory DEFAULT = new DefaultCodecFactory(/* videoEncoderSelector= */ null);
/** /**
* Returns a {@link Codec} for audio decoding. * Returns a {@link Codec} for audio decoding.
...@@ -76,7 +76,7 @@ public final class Codec { ...@@ -76,7 +76,7 @@ public final class Codec {
public interface EncoderFactory { public interface EncoderFactory {
/** A default {@code EncoderFactory} implementation. */ /** A default {@code EncoderFactory} implementation. */
EncoderFactory DEFAULT = new DefaultCodecFactory(); EncoderFactory DEFAULT = new DefaultCodecFactory(EncoderSelector.DEFAULT);
/** /**
* Returns a {@link Codec} for audio encoding. * Returns a {@link Codec} for audio encoding.
......
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultCodecFactory.java
...@@ -18,7 +18,9 @@ package com.google.android.exoplayer2.transformer; ...@@ -18,7 +18,9 @@ package com.google.android.exoplayer2.transformer;
import static com.google.android.exoplayer2.util.Assertions.checkArgument; import static com.google.android.exoplayer2.util.Assertions.checkArgument;
import static com.google.android.exoplayer2.util.Assertions.checkNotNull; import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
import static com.google.android.exoplayer2.util.Util.SDK_INT; import static com.google.android.exoplayer2.util.Util.SDK_INT;
import static java.lang.Math.abs;
import android.annotation.SuppressLint; import android.annotation.SuppressLint;
import android.media.MediaCodec; import android.media.MediaCodec;
...@@ -33,7 +35,9 @@ import com.google.android.exoplayer2.mediacodec.MediaCodecUtil; ...@@ -33,7 +35,9 @@ import com.google.android.exoplayer2.mediacodec.MediaCodecUtil;
import com.google.android.exoplayer2.util.MediaFormatUtil; import com.google.android.exoplayer2.util.MediaFormatUtil;
import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.MimeTypes;
import com.google.android.exoplayer2.util.TraceUtil; import com.google.android.exoplayer2.util.TraceUtil;
import com.google.common.collect.ImmutableList;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList;
import java.util.List; import java.util.List;
import org.checkerframework.checker.nullness.qual.RequiresNonNull; import org.checkerframework.checker.nullness.qual.RequiresNonNull;
...@@ -48,6 +52,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -48,6 +52,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private static final int DEFAULT_FRAME_RATE = 60; private static final int DEFAULT_FRAME_RATE = 60;
private static final int DEFAULT_I_FRAME_INTERVAL_SECS = 1; private static final int DEFAULT_I_FRAME_INTERVAL_SECS = 1;
@Nullable private final EncoderSelector videoEncoderSelector;
/** Creates a new instance. */
public DefaultCodecFactory(@Nullable EncoderSelector videoEncoderSelector) {
this.videoEncoderSelector = videoEncoderSelector;
}
@Override @Override
public Codec createForAudioDecoding(Format format) throws TransformationException { public Codec createForAudioDecoding(Format format) throws TransformationException {
MediaFormat mediaFormat = MediaFormat mediaFormat =
...@@ -60,6 +71,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -60,6 +71,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
return createCodec( return createCodec(
format, format,
mediaFormat, mediaFormat,
/* mediaCodecName= */ null,
/* isVideo= */ false, /* isVideo= */ false,
/* isDecoder= */ true, /* isDecoder= */ true,
/* outputSurface= */ null); /* outputSurface= */ null);
...@@ -83,12 +95,18 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -83,12 +95,18 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
} }
return createCodec( return createCodec(
format, mediaFormat, /* isVideo= */ true, /* isDecoder= */ true, outputSurface); format,
mediaFormat,
/* mediaCodecName= */ null,
/* isVideo= */ true,
/* isDecoder= */ true,
outputSurface);
} }
@Override @Override
public Codec createForAudioEncoding(Format format, List<String> allowedMimeTypes) public Codec createForAudioEncoding(Format format, List<String> allowedMimeTypes)
throws TransformationException { throws TransformationException {
// TODO(b/210591626) Add encoder selection for audio.
checkArgument(!allowedMimeTypes.isEmpty()); checkArgument(!allowedMimeTypes.isEmpty());
if (!allowedMimeTypes.contains(format.sampleMimeType)) { if (!allowedMimeTypes.contains(format.sampleMimeType)) {
// TODO(b/210591626): Pick fallback MIME type using same strategy as for encoder // TODO(b/210591626): Pick fallback MIME type using same strategy as for encoder
...@@ -103,6 +121,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -103,6 +121,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
return createCodec( return createCodec(
format, format,
mediaFormat, mediaFormat,
/* mediaCodecName= */ null,
/* isVideo= */ false, /* isVideo= */ false,
/* isDecoder= */ false, /* isDecoder= */ false,
/* outputSurface= */ null); /* outputSurface= */ null);
...@@ -118,11 +137,23 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -118,11 +137,23 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
checkArgument(format.height <= format.width); checkArgument(format.height <= format.width);
checkArgument(format.rotationDegrees == 0); checkArgument(format.rotationDegrees == 0);
checkNotNull(format.sampleMimeType); checkNotNull(format.sampleMimeType);
checkArgument(!allowedMimeTypes.isEmpty()); checkArgument(!allowedMimeTypes.isEmpty());
checkStateNotNull(videoEncoderSelector);
format = getVideoEncoderSupportedFormat(format, allowedMimeTypes); @Nullable
EncoderAndSupportedFormat encoderAndSupportedFormat =
findEncoderWithClosestFormatSupport(format, videoEncoderSelector, allowedMimeTypes);
if (encoderAndSupportedFormat == null) {
throw createTransformationException(
new IllegalArgumentException(
"No encoder available that supports the requested output format."),
format,
/* isVideo= */ true,
/* isDecoder= */ false,
/* mediaCodecName= */ null);
}
format = encoderAndSupportedFormat.supportedFormat;
MediaFormat mediaFormat = MediaFormat mediaFormat =
MediaFormat.createVideoFormat( MediaFormat.createVideoFormat(
checkNotNull(format.sampleMimeType), format.width, format.height); checkNotNull(format.sampleMimeType), format.width, format.height);
...@@ -144,6 +175,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -144,6 +175,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
return createCodec( return createCodec(
format, format,
mediaFormat, mediaFormat,
encoderAndSupportedFormat.encoderInfo.getName(),
/* isVideo= */ true, /* isVideo= */ true,
/* isDecoder= */ false, /* isDecoder= */ false,
/* outputSurface= */ null); /* outputSurface= */ null);
...@@ -153,6 +185,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -153,6 +185,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private static Codec createCodec( private static Codec createCodec(
Format format, Format format,
MediaFormat mediaFormat, MediaFormat mediaFormat,
@Nullable String mediaCodecName,
boolean isVideo, boolean isVideo,
boolean isDecoder, boolean isDecoder,
@Nullable Surface outputSurface) @Nullable Surface outputSurface)
...@@ -161,7 +194,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -161,7 +194,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
@Nullable Surface inputSurface = null; @Nullable Surface inputSurface = null;
try { try {
mediaCodec = mediaCodec =
isDecoder mediaCodecName != null
? MediaCodec.createByCodecName(mediaCodecName)
: isDecoder
? MediaCodec.createDecoderByType(format.sampleMimeType) ? MediaCodec.createDecoderByType(format.sampleMimeType)
: MediaCodec.createEncoderByType(format.sampleMimeType); : MediaCodec.createEncoderByType(format.sampleMimeType);
configureCodec(mediaCodec, mediaFormat, isDecoder, outputSurface); configureCodec(mediaCodec, mediaFormat, isDecoder, outputSurface);
...@@ -173,7 +208,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -173,7 +208,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
if (inputSurface != null) { if (inputSurface != null) {
inputSurface.release(); inputSurface.release();
} }
@Nullable String mediaCodecName = null;
if (mediaCodec != null) { if (mediaCodec != null) {
mediaCodecName = mediaCodec.getName(); mediaCodecName = mediaCodec.getName();
mediaCodec.release(); mediaCodec.release();
...@@ -203,80 +237,144 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -203,80 +237,144 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
TraceUtil.endSection(); TraceUtil.endSection();
} }
/**
* Finds the {@link EncoderAndSupportedFormat} whose {@link EncoderAndSupportedFormat#encoderInfo
* encoder} supports the {@code requestedFormat} most closely; {@code null} if none is found.
*/
@RequiresNonNull("#1.sampleMimeType") @RequiresNonNull("#1.sampleMimeType")
private static Format getVideoEncoderSupportedFormat( @Nullable
Format requestedFormat, List<String> allowedMimeTypes) throws TransformationException { private static EncoderAndSupportedFormat findEncoderWithClosestFormatSupport(
Format requestedFormat, EncoderSelector encoderSelector, List<String> allowedMimeTypes) {
String mimeType = requestedFormat.sampleMimeType; String mimeType = requestedFormat.sampleMimeType;
Format.Builder formatBuilder = requestedFormat.buildUpon();
// TODO(b/210591626) Implement encoder filtering. // TODO(b/210591626) Improve MIME type selection.
if (!allowedMimeTypes.contains(mimeType) List<MediaCodecInfo> encodersForMimeType = encoderSelector.selectEncoderInfos(mimeType);
|| EncoderUtil.getSupportedEncoders(mimeType).isEmpty()) { if (!allowedMimeTypes.contains(mimeType) || encodersForMimeType.isEmpty()) {
mimeType = mimeType =
allowedMimeTypes.contains(DEFAULT_FALLBACK_MIME_TYPE) allowedMimeTypes.contains(DEFAULT_FALLBACK_MIME_TYPE)
? DEFAULT_FALLBACK_MIME_TYPE ? DEFAULT_FALLBACK_MIME_TYPE
: allowedMimeTypes.get(0); : allowedMimeTypes.get(0);
if (EncoderUtil.getSupportedEncoders(mimeType).isEmpty()) { encodersForMimeType = encoderSelector.selectEncoderInfos(mimeType);
throw createTransformationException( if (encodersForMimeType.isEmpty()) {
new IllegalArgumentException( return null;
"No encoder is found for requested MIME type " + requestedFormat.sampleMimeType),
requestedFormat,
/* isVideo= */ true,
/* isDecoder= */ false,
/* mediaCodecName= */ null);
} }
} }
formatBuilder.setSampleMimeType(mimeType); String finalMimeType = mimeType;
MediaCodecInfo encoderInfo = EncoderUtil.getSupportedEncoders(mimeType).get(0); ImmutableList<MediaCodecInfo> filteredEncoders =
filterEncoders(
int width = requestedFormat.width; encodersForMimeType,
int height = requestedFormat.height; /* cost= */ (encoderInfo) -> {
@Nullable @Nullable
Pair<Integer, Integer> encoderSupportedResolution = Pair<Integer, Integer> closestSupportedResolution =
EncoderUtil.getClosestSupportedResolution(encoderInfo, mimeType, width, height); EncoderUtil.getClosestSupportedResolution(
if (encoderSupportedResolution == null) { encoderInfo, finalMimeType, requestedFormat.width, requestedFormat.height);
throw createTransformationException( if (closestSupportedResolution == null) {
new IllegalArgumentException( // Drops encoder.
"Cannot find fallback resolution for resolution " + width + " x " + height), return Integer.MAX_VALUE;
requestedFormat, }
/* isVideo= */ true, return abs(
/* isDecoder= */ false, requestedFormat.width * requestedFormat.height
/* mediaCodecName= */ null); - closestSupportedResolution.first * closestSupportedResolution.second);
});
if (filteredEncoders.isEmpty()) {
return null;
} }
width = encoderSupportedResolution.first; // The supported resolution is the same for all remaining encoders.
height = encoderSupportedResolution.second; Pair<Integer, Integer> finalResolution =
formatBuilder.setWidth(width).setHeight(height); checkNotNull(
EncoderUtil.getClosestSupportedResolution(
filteredEncoders.get(0),
finalMimeType,
requestedFormat.width,
requestedFormat.height));
// The frameRate does not affect the resulting frame rate. It affects the encoder's rate control int requestedBitrate =
// algorithm. Setting it too high may lead to video quality degradation. requestedFormat.averageBitrate == Format.NO_VALUE
float frameRate = ? getSuggestedBitrate(
requestedFormat.frameRate != Format.NO_VALUE /* width= */ finalResolution.first,
? requestedFormat.frameRate /* height= */ finalResolution.second,
: DEFAULT_FRAME_RATE; requestedFormat.frameRate == Format.NO_VALUE
int bitrate = ? DEFAULT_FRAME_RATE
: requestedFormat.frameRate)
: requestedFormat.averageBitrate;
filteredEncoders =
filterEncoders(
filteredEncoders,
/* cost= */ (encoderInfo) -> {
int achievableBitrate =
EncoderUtil.getClosestSupportedBitrate( EncoderUtil.getClosestSupportedBitrate(
encoderInfo, encoderInfo, finalMimeType, requestedBitrate);
mimeType, return abs(achievableBitrate - requestedBitrate);
/* bitrate= */ requestedFormat.averageBitrate != Format.NO_VALUE });
? requestedFormat.averageBitrate if (filteredEncoders.isEmpty()) {
: getSuggestedBitrate(width, height, frameRate)); return null;
formatBuilder.setFrameRate(frameRate).setAverageBitrate(bitrate); }
MediaCodecInfo pickedEncoder = filteredEncoders.get(0);
@Nullable @Nullable
Pair<Integer, Integer> profileLevel = MediaCodecUtil.getCodecProfileAndLevel(requestedFormat); Pair<Integer, Integer> profileLevel = MediaCodecUtil.getCodecProfileAndLevel(requestedFormat);
if (profileLevel == null @Nullable String codecs = null;
// Transcoding to another MIME type. if (profileLevel != null
|| !requestedFormat.sampleMimeType.equals(mimeType) && requestedFormat.sampleMimeType.equals(finalMimeType)
|| !EncoderUtil.isProfileLevelSupported( && EncoderUtil.isProfileLevelSupported(
encoderInfo, pickedEncoder,
mimeType, finalMimeType,
/* profile= */ profileLevel.first, /* profile= */ profileLevel.first,
/* level= */ profileLevel.second)) { /* level= */ profileLevel.second)) {
formatBuilder.setCodecs(null); codecs = requestedFormat.codecs;
}
Format encoderSupportedFormat =
requestedFormat
.buildUpon()
.setSampleMimeType(finalMimeType)
.setCodecs(codecs)
.setWidth(finalResolution.first)
.setHeight(finalResolution.second)
.setFrameRate(
requestedFormat.frameRate != Format.NO_VALUE
? requestedFormat.frameRate
: DEFAULT_FRAME_RATE)
.setAverageBitrate(
EncoderUtil.getClosestSupportedBitrate(
pickedEncoder, finalMimeType, requestedBitrate))
.build();
return new EncoderAndSupportedFormat(pickedEncoder, encoderSupportedFormat);
}
private interface EncoderFallbackCost {
/**
* Returns a cost that represents the gap between the requested encoding parameter(s) and the
* {@link MediaCodecInfo encoder}'s support for them.
*
* <p>The method must return {@link Integer#MAX_VALUE} when the {@link MediaCodecInfo encoder}
* does not support the encoding parameters.
*/
int getParameterSupportGap(MediaCodecInfo encoderInfo);
} }
return formatBuilder.build(); private static ImmutableList<MediaCodecInfo> filterEncoders(
List<MediaCodecInfo> encoders, EncoderFallbackCost cost) {
List<MediaCodecInfo> filteredEncoders = new ArrayList<>(encoders.size());
int minGap = Integer.MAX_VALUE;
for (int i = 0; i < encoders.size(); i++) {
MediaCodecInfo encoderInfo = encoders.get(i);
int gap = cost.getParameterSupportGap(encoderInfo);
if (gap == Integer.MAX_VALUE) {
continue;
}
if (gap < minGap) {
minGap = gap;
filteredEncoders.clear();
filteredEncoders.add(encoderInfo);
} else if (gap == minGap) {
filteredEncoders.add(encoderInfo);
}
}
return ImmutableList.copyOf(filteredEncoders);
} }
/** Computes the video bit rate using the Kush Gauge. */ /** Computes the video bit rate using the Kush Gauge. */
...@@ -315,4 +413,20 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ...@@ -315,4 +413,20 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
} }
return TransformationException.createForUnexpected(cause); return TransformationException.createForUnexpected(cause);
} }
/**
* A class wrapping a selected {@link MediaCodecInfo encoder} and its supported {@link Format}.
*/
private static class EncoderAndSupportedFormat {
/** The {@link MediaCodecInfo} that describes the encoder. */
public final MediaCodecInfo encoderInfo;
/** The {@link Format} that this encoder supports. */
public final Format supportedFormat;
/** Creates a new instance. */
public EncoderAndSupportedFormat(MediaCodecInfo encoderInfo, Format supportedFormat) {
this.encoderInfo = encoderInfo;
this.supportedFormat = supportedFormat;
}
}
} }
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/EncoderSelector.java 0 → 100644
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.transformer;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import com.google.android.exoplayer2.util.MimeTypes;
import java.util.List;
/** Selector of {@link MediaCodec} encoder instances. */
public interface EncoderSelector {
/**
* Default implementation of {@link EncoderSelector}, which returns the preferred encoders for the
* given {@link MimeTypes MIME type}.
*/
EncoderSelector DEFAULT = EncoderUtil::getSupportedEncoders;
/**
* Returns a list of encoders that can encode media in the specified {@code mimeType}, in priority
* order.
*
* @param mimeType The {@link MimeTypes MIME type} for which an encoder is required.
* @return An unmodifiable list of {@link MediaCodecInfo encoders} that supports the {@code
* mimeType}. The list may be empty.
*/
List<MediaCodecInfo> selectEncoderInfos(String mimeType);
}
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/EncoderUtil.java
...@@ -22,7 +22,11 @@ import android.media.MediaCodec; ...@@ -22,7 +22,11 @@ import android.media.MediaCodec;
import android.media.MediaCodecInfo; import android.media.MediaCodecInfo;
import android.media.MediaCodecList; import android.media.MediaCodecList;
import android.util.Pair; import android.util.Pair;
import androidx.annotation.DoNotInline;
import androidx.annotation.Nullable; import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import com.google.android.exoplayer2.util.MimeTypes;
import com.google.android.exoplayer2.util.Util;
import com.google.common.base.Ascii; import com.google.common.base.Ascii;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import java.util.ArrayList; import java.util.ArrayList;
...@@ -128,6 +132,42 @@ public final class EncoderUtil { ...@@ -128,6 +132,42 @@ public final class EncoderUtil {
.clamp(bitrate); .clamp(bitrate);
} }
/** Checks if a {@link MediaCodecInfo codec} is hardware-accelerated. */
public static boolean isHardwareAccelerated(MediaCodecInfo encoderInfo, String mimeType) {
// TODO(b/214964116): Merge into MediaCodecUtil.
if (Util.SDK_INT >= 29) {
return Api29.isHardwareAccelerated(encoderInfo);
}
// codecInfo.isHardwareAccelerated() == !codecInfo.isSoftwareOnly() is not necessarily true.
// However, we assume this to be true as an approximation.
return !isSoftwareOnly(encoderInfo, mimeType);
}
private static boolean isSoftwareOnly(MediaCodecInfo encoderInfo, String mimeType) {
if (Util.SDK_INT >= 29) {
return Api29.isSoftwareOnly(encoderInfo);
}
if (MimeTypes.isAudio(mimeType)) {
// Assume audio decoders are software only.
return true;
}
String codecName = Ascii.toLowerCase(encoderInfo.getName());
if (codecName.startsWith("arc.")) {
// App Runtime for Chrome (ARC) codecs
return false;
}
// Estimate whether a codec is software-only, to emulate isSoftwareOnly on API < 29.
return codecName.startsWith("omx.google.")
|| codecName.startsWith("omx.ffmpeg.")
|| (codecName.startsWith("omx.sec.") && codecName.contains(".sw."))
|| codecName.equals("omx.qcom.video.decoder.hevcswvdec")
|| codecName.startsWith("c2.android.")
|| codecName.startsWith("c2.google.")
|| (!codecName.startsWith("omx.") && !codecName.startsWith("c2."));
}
/** /**
* Align to the closest resolution that respects the encoder's supported alignment. * Align to the closest resolution that respects the encoder's supported alignment.
* *
...@@ -152,5 +192,18 @@ public final class EncoderUtil { ...@@ -152,5 +192,18 @@ public final class EncoderUtil {
} }
} }
@RequiresApi(29)
private static final class Api29 {
@DoNotInline
public static boolean isHardwareAccelerated(MediaCodecInfo encoderInfo) {
return encoderInfo.isHardwareAccelerated();
}
@DoNotInline
public static boolean isSoftwareOnly(MediaCodecInfo encoderInfo) {
return encoderInfo.isSoftwareOnly();
}
}
private EncoderUtil() {} private EncoderUtil() {}
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment