Skip to content

SDK / exoplayer

Go to a project
Commit 05c5fe23 by huangdarwin Committed by Tofunmi Adigun-Hameed
Options

ExoPlayer: Add setVideoFrameProcessorFactory(). 

This allows apps to use a custom VideoFrameProcessor implementation for video
playback. This may be useful, for example, when outputting to a texture.

PiperOrigin-RevId: 534044831
(cherry picked from commit 703923d196934fdcc019ffd610b6f778f8fa043d)
parent a80b1675
Inline Side-by-side
Showing with 123 additions and 63 deletions
library/core/src/main/java/com/google/android/exoplayer2/ExoPlayer.java
......@@ -61,6 +61,7 @@ import com.google.android.exoplayer2.util.Clock;
import com.google.android.exoplayer2.util.Effect;
import com.google.android.exoplayer2.util.PriorityTaskManager;
import com.google.android.exoplayer2.util.Util;
import com.google.android.exoplayer2.util.VideoFrameProcessor;
import com.google.android.exoplayer2.video.MediaCodecVideoRenderer;
import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
import com.google.android.exoplayer2.video.VideoSize;
......@@ -1488,6 +1489,18 @@ public interface ExoPlayer extends Player {
void setVideoEffects(List<Effect> videoEffects);
/**
* Sets a {@link VideoFrameProcessor.Factory} to create the {@link VideoFrameProcessor} that
* applies video effects set in {@link #setVideoEffects}.
*
* <p>See {@link #setVideoEffects} for limitations.
*
* @param videoFrameProcessorFactory The {@link VideoFrameProcessor.Factory} to use to apply the
* video effects.
*/
@RequiresApi(18)
void setVideoFrameProcessorFactory(VideoFrameProcessor.Factory videoFrameProcessorFactory);
/**
* Sets the {@link C.VideoScalingMode}.
*
* <p>The scaling mode only applies if a {@link MediaCodec}-based video {@link Renderer} is
......
library/core/src/main/java/com/google/android/exoplayer2/ExoPlayerImpl.java
......@@ -28,6 +28,7 @@ import static com.google.android.exoplayer2.Renderer.MSG_SET_SCALING_MODE;
import static com.google.android.exoplayer2.Renderer.MSG_SET_SKIP_SILENCE_ENABLED;
import static com.google.android.exoplayer2.Renderer.MSG_SET_VIDEO_EFFECTS;
import static com.google.android.exoplayer2.Renderer.MSG_SET_VIDEO_FRAME_METADATA_LISTENER;
import static com.google.android.exoplayer2.Renderer.MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY;
import static com.google.android.exoplayer2.Renderer.MSG_SET_VIDEO_OUTPUT;
import static com.google.android.exoplayer2.Renderer.MSG_SET_VIDEO_OUTPUT_RESOLUTION;
import static com.google.android.exoplayer2.Renderer.MSG_SET_VOLUME;
......@@ -94,6 +95,7 @@ import com.google.android.exoplayer2.util.Log;
import com.google.android.exoplayer2.util.PriorityTaskManager;
import com.google.android.exoplayer2.util.Size;
import com.google.android.exoplayer2.util.Util;
import com.google.android.exoplayer2.util.VideoFrameProcessor;
import com.google.android.exoplayer2.video.VideoDecoderOutputBufferRenderer;
import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
import com.google.android.exoplayer2.video.VideoRendererEventListener;
......@@ -1228,6 +1230,14 @@ import java.util.concurrent.TimeoutException;
}
@Override
public void setVideoFrameProcessorFactory(
VideoFrameProcessor.Factory videoFrameProcessorFactory) {
verifyApplicationThread();
sendRendererMessage(
TRACK_TYPE_VIDEO, MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY, videoFrameProcessorFactory);
}
@Override
public void setVideoScalingMode(@C.VideoScalingMode int videoScalingMode) {
verifyApplicationThread();
this.videoScalingMode = videoScalingMode;
......
library/core/src/main/java/com/google/android/exoplayer2/Renderer.java
......@@ -29,6 +29,7 @@ import com.google.android.exoplayer2.util.Effect;
import com.google.android.exoplayer2.util.MediaClock;
import com.google.android.exoplayer2.util.Size;
import com.google.android.exoplayer2.util.Util;
import com.google.android.exoplayer2.util.VideoFrameProcessor;
import com.google.android.exoplayer2.video.VideoDecoderOutputBufferRenderer;
import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
import com.google.android.exoplayer2.video.spherical.CameraMotionListener;
......@@ -86,9 +87,9 @@ public interface Renderer extends PlayerMessage.Target {
* #MSG_SET_SCALING_MODE}, {@link #MSG_SET_CHANGE_FRAME_RATE_STRATEGY}, {@link
* #MSG_SET_AUX_EFFECT_INFO}, {@link #MSG_SET_VIDEO_FRAME_METADATA_LISTENER}, {@link
* #MSG_SET_CAMERA_MOTION_LISTENER}, {@link #MSG_SET_SKIP_SILENCE_ENABLED}, {@link
* #MSG_SET_AUDIO_SESSION_ID}, {@link #MSG_SET_WAKEUP_LISTENER}, {@link #MSG_SET_VIDEO_EFFECTS} or
* {@link #MSG_SET_VIDEO_OUTPUT_RESOLUTION}. May also be an app-defined value (see {@link
* #MSG_CUSTOM_BASE}).
* #MSG_SET_AUDIO_SESSION_ID}, {@link #MSG_SET_WAKEUP_LISTENER}, {@link #MSG_SET_VIDEO_EFFECTS},
* {@link #MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY} or {@link #MSG_SET_VIDEO_OUTPUT_RESOLUTION}. May
* also be an app-defined value (see {@link #MSG_CUSTOM_BASE}).
*/
@Documented
@Retention(RetentionPolicy.SOURCE)
......@@ -108,6 +109,7 @@ public interface Renderer extends PlayerMessage.Target {
MSG_SET_AUDIO_SESSION_ID,
MSG_SET_WAKEUP_LISTENER,
MSG_SET_VIDEO_EFFECTS,
MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY,
MSG_SET_VIDEO_OUTPUT_RESOLUTION
})
public @interface MessageType {}
......@@ -216,11 +218,16 @@ public interface Renderer extends PlayerMessage.Target {
*/
int MSG_SET_VIDEO_EFFECTS = 13;
/**
* The type of a message that can be passed to a video renderer. The message payload should be a
* {@link VideoFrameProcessor.Factory}.
*/
int MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY = 14;
/**
* The type of a message that can be passed to a video renderer to set the desired output
* resolution. The message payload should be a {@link Size} of the desired output width and
* height. Use this method only when playing with video {@linkplain Effect effects}.
*/
int MSG_SET_VIDEO_OUTPUT_RESOLUTION = 14;
int MSG_SET_VIDEO_OUTPUT_RESOLUTION = 15;
/**
* Applications or extensions may define custom {@code MSG_*} constants that can be passed to
* renderers. These custom constants must be greater than or equal to this value.
......
library/core/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java
......@@ -48,6 +48,7 @@ import com.google.android.exoplayer2.util.ConditionVariable;
import com.google.android.exoplayer2.util.Effect;
import com.google.android.exoplayer2.util.PriorityTaskManager;
import com.google.android.exoplayer2.util.Size;
import com.google.android.exoplayer2.util.VideoFrameProcessor;
import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
import com.google.android.exoplayer2.video.VideoSize;
import com.google.android.exoplayer2.video.spherical.CameraMotionListener;
......@@ -661,6 +662,13 @@ public class SimpleExoPlayer extends BasePlayer
}
@Override
public void setVideoFrameProcessorFactory(
VideoFrameProcessor.Factory videoFrameProcessorFactory) {
blockUntilConstructorFinished();
player.setVideoFrameProcessorFactory(videoFrameProcessorFactory);
}
@Override
public void setSkipSilenceEnabled(boolean skipSilenceEnabled) {
blockUntilConstructorFinished();
player.setSkipSilenceEnabled(skipSilenceEnabled);
......
library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java
......@@ -683,6 +683,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
List<Effect> videoEffects = (List<Effect>) checkNotNull(message);
videoFrameProcessorManager.setVideoEffects(videoEffects);
break;
case MSG_SET_VIDEO_FRAME_PROCESSOR_FACTORY:
VideoFrameProcessor.Factory videoFrameProcessorFactory =
(VideoFrameProcessor.Factory) checkNotNull(message);
videoFrameProcessorManager.setVideoFrameProcessorFactory(videoFrameProcessorFactory);
break;
case MSG_SET_VIDEO_OUTPUT_RESOLUTION:
Size outputResolution = (Size) checkNotNull(message);
if (outputResolution.getWidth() != 0
......@@ -1872,6 +1877,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
private final ArrayDeque<Pair<Long, Format>> pendingFrameFormats;
private @MonotonicNonNull Handler handler;
private VideoFrameProcessor.@MonotonicNonNull Factory videoFrameProcessorFactory;
@Nullable private VideoFrameProcessor videoFrameProcessor;
@Nullable private CopyOnWriteArrayList<Effect> videoEffects;
@Nullable private Format inputFormat;
......@@ -1933,6 +1939,12 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
this.videoEffects.addAll(videoEffects);
}
/** Sets the {@link VideoFrameProcessor.Factory}. */
public void setVideoFrameProcessorFactory(
VideoFrameProcessor.Factory videoFrameProcessorFactory) {
this.videoFrameProcessorFactory = videoFrameProcessorFactory;
}
/** Returns whether video frame processing is enabled. */
public boolean isEnabled() {
return videoFrameProcessor != null;
......@@ -2005,66 +2017,69 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
VideoFrameProcessorAccessor.createRotationEffect(inputFormat.rotationDegrees));
}
videoFrameProcessorFactory =
videoFrameProcessorFactory == null
? VideoFrameProcessorAccessor.getFrameProcessorFactory()
: videoFrameProcessorFactory;
videoFrameProcessor =
VideoFrameProcessorAccessor.getFrameProcessorFactory()
.create(
renderer.context,
checkNotNull(videoEffects),
DebugViewProvider.NONE,
inputAndOutputColorInfos.first,
inputAndOutputColorInfos.second,
/* renderFramesAutomatically= */ false,
/* listenerExecutor= */ handler::post,
new VideoFrameProcessor.Listener() {
@Override
public void onOutputSizeChanged(int width, int height) {
@Nullable Format inputFormat = VideoFrameProcessorManager.this.inputFormat;
checkStateNotNull(inputFormat);
// TODO(b/264889146): Handle Effect that changes output size based on pts.
processedFrameSize =
new VideoSize(
width,
height,
// VideoFrameProcessor is configured to produce rotation free
// frames.
/* unappliedRotationDegrees= */ 0,
// VideoFrameProcessor always outputs pixelWidthHeightRatio 1.
/* pixelWidthHeightRatio= */ 1.f);
pendingOutputSizeChange = true;
}
@Override
public void onOutputFrameAvailableForRendering(long presentationTimeUs) {
if (registeredLastFrame) {
checkState(lastCodecBufferPresentationTimestampUs != C.TIME_UNSET);
}
processedFramesTimestampsUs.add(presentationTimeUs);
// TODO(b/257464707) Support extensively modified media.
if (registeredLastFrame
&& presentationTimeUs >= lastCodecBufferPresentationTimestampUs) {
processedLastFrame = true;
}
if (pendingOutputSizeChange) {
// Report the size change on releasing this frame.
pendingOutputSizeChange = false;
pendingOutputSizeChangeNotificationTimeUs = presentationTimeUs;
}
}
@Override
public void onError(VideoFrameProcessingException exception) {
renderer.setPendingPlaybackException(
renderer.createRendererException(
exception,
inputFormat,
PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED));
}
@Override
public void onEnded() {
throw new IllegalStateException();
}
});
videoFrameProcessorFactory.create(
renderer.context,
checkNotNull(videoEffects),
DebugViewProvider.NONE,
inputAndOutputColorInfos.first,
inputAndOutputColorInfos.second,
/* renderFramesAutomatically= */ false,
/* listenerExecutor= */ handler::post,
new VideoFrameProcessor.Listener() {
@Override
public void onOutputSizeChanged(int width, int height) {
@Nullable Format inputFormat = VideoFrameProcessorManager.this.inputFormat;
checkStateNotNull(inputFormat);
// TODO(b/264889146): Handle Effect that changes output size based on pts.
processedFrameSize =
new VideoSize(
width,
height,
// VideoFrameProcessor is configured to produce rotation free
// frames.
/* unappliedRotationDegrees= */ 0,
// VideoFrameProcessor always outputs pixelWidthHeightRatio 1.
/* pixelWidthHeightRatio= */ 1.f);
pendingOutputSizeChange = true;
}
@Override
public void onOutputFrameAvailableForRendering(long presentationTimeUs) {
if (registeredLastFrame) {
checkState(lastCodecBufferPresentationTimestampUs != C.TIME_UNSET);
}
processedFramesTimestampsUs.add(presentationTimeUs);
// TODO(b/257464707) Support extensively modified media.
if (registeredLastFrame
&& presentationTimeUs >= lastCodecBufferPresentationTimestampUs) {
processedLastFrame = true;
}
if (pendingOutputSizeChange) {
// Report the size change on releasing this frame.
pendingOutputSizeChange = false;
pendingOutputSizeChangeNotificationTimeUs = presentationTimeUs;
}
}
@Override
public void onError(VideoFrameProcessingException exception) {
renderer.setPendingPlaybackException(
renderer.createRendererException(
exception,
inputFormat,
PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED));
}
@Override
public void onEnded() {
throw new IllegalStateException();
}
});
videoFrameProcessor.registerInputStream(VideoFrameProcessor.INPUT_TYPE_SURFACE);
this.initialStreamOffsetUs = initialStreamOffsetUs;
} catch (Exception e) {
......
testutils/src/main/java/com/google/android/exoplayer2/testutil/StubExoPlayer.java
......@@ -38,6 +38,7 @@ import com.google.android.exoplayer2.trackselection.TrackSelector;
import com.google.android.exoplayer2.util.Clock;
import com.google.android.exoplayer2.util.Effect;
import com.google.android.exoplayer2.util.PriorityTaskManager;
import com.google.android.exoplayer2.util.VideoFrameProcessor;
import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
import com.google.android.exoplayer2.video.spherical.CameraMotionListener;
import java.util.List;
......@@ -248,6 +249,12 @@ public class StubExoPlayer extends StubPlayer implements ExoPlayer {
}
@Override
public void setVideoFrameProcessorFactory(
VideoFrameProcessor.Factory videoFrameProcessorFactory) {
throw new UnsupportedOperationException();
}
@Override
public void setVideoScalingMode(int videoScalingMode) {
throw new UnsupportedOperationException();
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment