Skip to content

SDK / exoplayer

Go to a project
Commit 9473fda0 by christosts Committed by kim-vde
Options

Synchronize codec interaction with buffer queueing 

Add experimental method to synchronize MediaCodec interactions
with asynchronous queueing. When the feature is enabled, interactions
such as MediaCodec.setOutputSurface() triggered by the
MediaCodecRenderer will wait until all input buffers pending queueing
are first submitted to the MediaCodec.

PiperOrigin-RevId: 341423837
parent 86ae7eba
Inline Side-by-side
Showing with 119 additions and 40 deletions
library/core/src/main/java/com/google/android/exoplayer2/DefaultRenderersFactory.java
...@@ -92,6 +92,7 @@ public class DefaultRenderersFactory implements RenderersFactory { ...@@ -92,6 +92,7 @@ public class DefaultRenderersFactory implements RenderersFactory {
private boolean enableDecoderFallback; private boolean enableDecoderFallback;
private MediaCodecSelector mediaCodecSelector; private MediaCodecSelector mediaCodecSelector;
private boolean enableAsyncQueueing; private boolean enableAsyncQueueing;
private boolean enableSynchronizeCodecInteractionsWithQueueing;
private boolean enableFloatOutput; private boolean enableFloatOutput;
private boolean enableAudioTrackPlaybackParams; private boolean enableAudioTrackPlaybackParams;
private boolean enableOffload; private boolean enableOffload;
...@@ -155,12 +156,27 @@ public class DefaultRenderersFactory implements RenderersFactory { ...@@ -155,12 +156,27 @@ public class DefaultRenderersFactory implements RenderersFactory {
* @param enabled Whether asynchronous queueing is enabled. * @param enabled Whether asynchronous queueing is enabled.
* @return This factory, for convenience. * @return This factory, for convenience.
*/ */
public DefaultRenderersFactory experimentalEnableAsynchronousBufferQueueing(boolean enabled) { public DefaultRenderersFactory experimentalSetAsynchronousBufferQueueingEnabled(boolean enabled) {
enableAsyncQueueing = enabled; enableAsyncQueueing = enabled;
return this; return this;
} }
/** /**
* Enable synchronizing codec interactions with asynchronous buffer queueing.
*
* <p>This method is experimental, and will be renamed or removed in a future release.
*
* @param enabled Whether codec interactions will be synchronized with asynchronous buffer
* queueing.
* @return This factory, for convenience.
*/
public DefaultRenderersFactory experimentalSetSynchronizeCodecInteractionsWithQueueingEnabled(
boolean enabled) {
enableSynchronizeCodecInteractionsWithQueueing = enabled;
return this;
}
/**
* Sets whether to enable fallback to lower-priority decoders if decoder initialization fails. * Sets whether to enable fallback to lower-priority decoders if decoder initialization fails.
* This may result in using a decoder that is less efficient or slower than the primary decoder. * This may result in using a decoder that is less efficient or slower than the primary decoder.
* *
...@@ -336,7 +352,9 @@ public class DefaultRenderersFactory implements RenderersFactory { ...@@ -336,7 +352,9 @@ public class DefaultRenderersFactory implements RenderersFactory {
eventHandler, eventHandler,
eventListener, eventListener,
MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY); MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY);
videoRenderer.experimentalEnableAsynchronousBufferQueueing(enableAsyncQueueing); videoRenderer.experimentalSetAsynchronousBufferQueueingEnabled(enableAsyncQueueing);
videoRenderer.experimentalSetSynchronizeCodecInteractionsWithQueueingEnabled(
enableSynchronizeCodecInteractionsWithQueueing);
out.add(videoRenderer); out.add(videoRenderer);
if (extensionRendererMode == EXTENSION_RENDERER_MODE_OFF) { if (extensionRendererMode == EXTENSION_RENDERER_MODE_OFF) {
...@@ -461,7 +479,9 @@ public class DefaultRenderersFactory implements RenderersFactory { ...@@ -461,7 +479,9 @@ public class DefaultRenderersFactory implements RenderersFactory {
eventHandler, eventHandler,
eventListener, eventListener,
audioSink); audioSink);
audioRenderer.experimentalEnableAsynchronousBufferQueueing(enableAsyncQueueing); audioRenderer.experimentalSetAsynchronousBufferQueueingEnabled(enableAsyncQueueing);
audioRenderer.experimentalSetSynchronizeCodecInteractionsWithQueueingEnabled(
enableSynchronizeCodecInteractionsWithQueueing);
out.add(audioRenderer); out.add(audioRenderer);
if (extensionRendererMode == EXTENSION_RENDERER_MODE_OFF) { if (extensionRendererMode == EXTENSION_RENDERER_MODE_OFF) {
......
library/core/src/main/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecAdapter.java
...@@ -56,6 +56,7 @@ import java.nio.ByteBuffer; ...@@ -56,6 +56,7 @@ import java.nio.ByteBuffer;
private final MediaCodec codec; private final MediaCodec codec;
private final AsynchronousMediaCodecCallback asynchronousMediaCodecCallback; private final AsynchronousMediaCodecCallback asynchronousMediaCodecCallback;
private final AsynchronousMediaCodecBufferEnqueuer bufferEnqueuer; private final AsynchronousMediaCodecBufferEnqueuer bufferEnqueuer;
private final boolean synchronizeCodecInteractionsWithQueueing;
private boolean codecReleased; private boolean codecReleased;
@State private int state; @State private int state;
...@@ -65,20 +66,30 @@ import java.nio.ByteBuffer; ...@@ -65,20 +66,30 @@ import java.nio.ByteBuffer;
* @param codec The {@link MediaCodec} to wrap. * @param codec The {@link MediaCodec} to wrap.
* @param trackType One of {@link C#TRACK_TYPE_AUDIO} or {@link C#TRACK_TYPE_VIDEO}. Used for * @param trackType One of {@link C#TRACK_TYPE_AUDIO} or {@link C#TRACK_TYPE_VIDEO}. Used for
* labelling the internal thread accordingly. * labelling the internal thread accordingly.
* @param synchronizeCodecInteractionsWithQueueing Whether the adapter should synchronize {@link
* MediaCodec} interactions with asynchronous buffer queueing. When {@code true}, codec
* interactions will wait until all input buffers pending queueing wil be submitted to the
* {@link MediaCodec}.
*/ */
/* package */ AsynchronousMediaCodecAdapter(MediaCodec codec, int trackType) { /* package */ AsynchronousMediaCodecAdapter(
MediaCodec codec, int trackType, boolean synchronizeCodecInteractionsWithQueueing) {
this( this(
codec, codec,
new HandlerThread(createCallbackThreadLabel(trackType)), new HandlerThread(createCallbackThreadLabel(trackType)),
new HandlerThread(createQueueingThreadLabel(trackType))); new HandlerThread(createQueueingThreadLabel(trackType)),
synchronizeCodecInteractionsWithQueueing);
} }
@VisibleForTesting @VisibleForTesting
/* package */ AsynchronousMediaCodecAdapter( /* package */ AsynchronousMediaCodecAdapter(
MediaCodec codec, HandlerThread callbackThread, HandlerThread enqueueingThread) { MediaCodec codec,
HandlerThread callbackThread,
HandlerThread enqueueingThread,
boolean synchronizeCodecInteractionsWithQueueing) {
this.codec = codec; this.codec = codec;
this.asynchronousMediaCodecCallback = new AsynchronousMediaCodecCallback(callbackThread); this.asynchronousMediaCodecCallback = new AsynchronousMediaCodecCallback(callbackThread);
this.bufferEnqueuer = new AsynchronousMediaCodecBufferEnqueuer(codec, enqueueingThread); this.bufferEnqueuer = new AsynchronousMediaCodecBufferEnqueuer(codec, enqueueingThread);
this.synchronizeCodecInteractionsWithQueueing = synchronizeCodecInteractionsWithQueueing;
this.state = STATE_CREATED; this.state = STATE_CREATED;
} }
...@@ -181,6 +192,7 @@ import java.nio.ByteBuffer; ...@@ -181,6 +192,7 @@ import java.nio.ByteBuffer;
@Override @Override
public void setOnFrameRenderedListener(OnFrameRenderedListener listener, Handler handler) { public void setOnFrameRenderedListener(OnFrameRenderedListener listener, Handler handler) {
maybeBlockOnQueueing();
codec.setOnFrameRenderedListener( codec.setOnFrameRenderedListener(
(codec, presentationTimeUs, nanoTime) -> (codec, presentationTimeUs, nanoTime) ->
listener.onFrameRendered( listener.onFrameRendered(
...@@ -190,16 +202,19 @@ import java.nio.ByteBuffer; ...@@ -190,16 +202,19 @@ import java.nio.ByteBuffer;
@Override @Override
public void setOutputSurface(Surface surface) { public void setOutputSurface(Surface surface) {
maybeBlockOnQueueing();
codec.setOutputSurface(surface); codec.setOutputSurface(surface);
} }
@Override @Override
public void setParameters(Bundle params) { public void setParameters(Bundle params) {
maybeBlockOnQueueing();
codec.setParameters(params); codec.setParameters(params);
} }
@Override @Override
public void setVideoScalingMode(@VideoScalingMode int scalingMode) { public void setVideoScalingMode(@VideoScalingMode int scalingMode) {
maybeBlockOnQueueing();
codec.setVideoScalingMode(scalingMode); codec.setVideoScalingMode(scalingMode);
} }
...@@ -213,6 +228,19 @@ import java.nio.ByteBuffer; ...@@ -213,6 +228,19 @@ import java.nio.ByteBuffer;
asynchronousMediaCodecCallback.onOutputFormatChanged(codec, format); asynchronousMediaCodecCallback.onOutputFormatChanged(codec, format);
} }
private void maybeBlockOnQueueing() {
if (synchronizeCodecInteractionsWithQueueing) {
try {
bufferEnqueuer.waitUntilQueueingComplete();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
// The playback thread should not be interrupted. Raising this as an
// IllegalStateException.
throw new IllegalStateException(e);
}
}
}
private static String createCallbackThreadLabel(int trackType) { private static String createCallbackThreadLabel(int trackType) {
return createThreadLabel(trackType, /* prefix= */ "ExoPlayer:MediaCodecAsyncAdapter:"); return createThreadLabel(trackType, /* prefix= */ "ExoPlayer:MediaCodecAsyncAdapter:");
} }
......
library/core/src/main/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecBufferEnqueuer.java
...@@ -17,6 +17,7 @@ ...@@ -17,6 +17,7 @@
package com.google.android.exoplayer2.mediacodec; package com.google.android.exoplayer2.mediacodec;
import static com.google.android.exoplayer2.util.Assertions.checkNotNull; import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import static com.google.android.exoplayer2.util.Util.castNonNull;
import android.media.MediaCodec; import android.media.MediaCodec;
import android.os.Handler; import android.os.Handler;
...@@ -46,7 +47,7 @@ class AsynchronousMediaCodecBufferEnqueuer { ...@@ -46,7 +47,7 @@ class AsynchronousMediaCodecBufferEnqueuer {
private static final int MSG_QUEUE_INPUT_BUFFER = 0; private static final int MSG_QUEUE_INPUT_BUFFER = 0;
private static final int MSG_QUEUE_SECURE_INPUT_BUFFER = 1; private static final int MSG_QUEUE_SECURE_INPUT_BUFFER = 1;
private static final int MSG_FLUSH = 2; private static final int MSG_OPEN_CV = 2;
@GuardedBy("MESSAGE_PARAMS_INSTANCE_POOL") @GuardedBy("MESSAGE_PARAMS_INSTANCE_POOL")
private static final ArrayDeque<MessageParams> MESSAGE_PARAMS_INSTANCE_POOL = new ArrayDeque<>(); private static final ArrayDeque<MessageParams> MESSAGE_PARAMS_INSTANCE_POOL = new ArrayDeque<>();
...@@ -110,8 +111,7 @@ class AsynchronousMediaCodecBufferEnqueuer { ...@@ -110,8 +111,7 @@ class AsynchronousMediaCodecBufferEnqueuer {
maybeThrowException(); maybeThrowException();
MessageParams messageParams = getMessageParams(); MessageParams messageParams = getMessageParams();
messageParams.setQueueParams(index, offset, size, presentationTimeUs, flags); messageParams.setQueueParams(index, offset, size, presentationTimeUs, flags);
Message message = Message message = castNonNull(handler).obtainMessage(MSG_QUEUE_INPUT_BUFFER, messageParams);
Util.castNonNull(handler).obtainMessage(MSG_QUEUE_INPUT_BUFFER, messageParams);
message.sendToTarget(); message.sendToTarget();
} }
...@@ -131,7 +131,7 @@ class AsynchronousMediaCodecBufferEnqueuer { ...@@ -131,7 +131,7 @@ class AsynchronousMediaCodecBufferEnqueuer {
messageParams.setQueueParams(index, offset, /* size= */ 0, presentationTimeUs, flags); messageParams.setQueueParams(index, offset, /* size= */ 0, presentationTimeUs, flags);
copy(info, messageParams.cryptoInfo); copy(info, messageParams.cryptoInfo);
Message message = Message message =
Util.castNonNull(handler).obtainMessage(MSG_QUEUE_SECURE_INPUT_BUFFER, messageParams); castNonNull(handler).obtainMessage(MSG_QUEUE_SECURE_INPUT_BUFFER, messageParams);
message.sendToTarget(); message.sendToTarget();
} }
...@@ -158,6 +158,11 @@ class AsynchronousMediaCodecBufferEnqueuer { ...@@ -158,6 +158,11 @@ class AsynchronousMediaCodecBufferEnqueuer {
started = false; started = false;
} }
/** Blocks the current thread until all input buffers pending queueing are submitted. */
public void waitUntilQueueingComplete() throws InterruptedException {
blockUntilHandlerThreadIsIdle();
}
private void maybeThrowException() { private void maybeThrowException() {
@Nullable RuntimeException exception = pendingRuntimeException.getAndSet(null); @Nullable RuntimeException exception = pendingRuntimeException.getAndSet(null);
if (exception != null) { if (exception != null) {
...@@ -170,15 +175,19 @@ class AsynchronousMediaCodecBufferEnqueuer { ...@@ -170,15 +175,19 @@ class AsynchronousMediaCodecBufferEnqueuer {
* blocks until the {@link #handlerThread} is idle. * blocks until the {@link #handlerThread} is idle.
*/ */
private void flushHandlerThread() throws InterruptedException { private void flushHandlerThread() throws InterruptedException {
Handler handler = Util.castNonNull(this.handler); Handler handler = castNonNull(this.handler);
handler.removeCallbacksAndMessages(null); handler.removeCallbacksAndMessages(null);
conditionVariable.close(); blockUntilHandlerThreadIsIdle();
handler.obtainMessage(MSG_FLUSH).sendToTarget();
conditionVariable.block();
// Check if any exceptions happened during the last queueing action. // Check if any exceptions happened during the last queueing action.
maybeThrowException(); maybeThrowException();
} }
private void blockUntilHandlerThreadIsIdle() throws InterruptedException {
conditionVariable.close();
castNonNull(handler).obtainMessage(MSG_OPEN_CV).sendToTarget();
conditionVariable.block();
}
// Called from the handler thread // Called from the handler thread
@VisibleForTesting @VisibleForTesting
...@@ -203,7 +212,7 @@ class AsynchronousMediaCodecBufferEnqueuer { ...@@ -203,7 +212,7 @@ class AsynchronousMediaCodecBufferEnqueuer {
params.presentationTimeUs, params.presentationTimeUs,
params.flags); params.flags);
break; break;
case MSG_FLUSH: case MSG_OPEN_CV:
conditionVariable.open(); conditionVariable.open();
break; break;
default: default:
......
library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java
...@@ -348,6 +348,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { ...@@ -348,6 +348,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
private boolean waitingForFirstSampleInFormat; private boolean waitingForFirstSampleInFormat;
private boolean pendingOutputEndOfStream; private boolean pendingOutputEndOfStream;
private boolean enableAsynchronousBufferQueueing; private boolean enableAsynchronousBufferQueueing;
private boolean enableSynchronizeCodecInteractionsWithQueueing;
@Nullable private ExoPlaybackException pendingPlaybackException; @Nullable private ExoPlaybackException pendingPlaybackException;
protected DecoderCounters decoderCounters; protected DecoderCounters decoderCounters;
private long outputStreamStartPositionUs; private long outputStreamStartPositionUs;
...@@ -412,10 +413,24 @@ public abstract class MediaCodecRenderer extends BaseRenderer { ...@@ -412,10 +413,24 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
* <p>This method is experimental, and will be renamed or removed in a future release. It should * <p>This method is experimental, and will be renamed or removed in a future release. It should
* only be called before the renderer is used. * only be called before the renderer is used.
*/ */
public void experimentalEnableAsynchronousBufferQueueing(boolean enabled) { public void experimentalSetAsynchronousBufferQueueingEnabled(boolean enabled) {
enableAsynchronousBufferQueueing = enabled; enableAsynchronousBufferQueueing = enabled;
} }
/**
* Enable synchronizing codec interactions with asynchronous buffer queueing.
*
* <p>When enabled, codec interactions will wait until all input buffers pending for asynchronous
* queueing are submitted to the {@link MediaCodec} first. This method is effective only if {@link
* #experimentalSetAsynchronousBufferQueueingEnabled asynchronous buffer queueing} is enabled.
*
* <p>This method is experimental, and will be renamed or removed in a future release. It should
* only be called before the renderer is used.
*/
public void experimentalSetSynchronizeCodecInteractionsWithQueueingEnabled(boolean enabled) {
enableSynchronizeCodecInteractionsWithQueueing = enabled;
}
@Override @Override
@AdaptiveSupport @AdaptiveSupport
public final int supportsMixedMimeTypeAdaptation() { public final int supportsMixedMimeTypeAdaptation() {
...@@ -1051,7 +1066,9 @@ public abstract class MediaCodecRenderer extends BaseRenderer { ...@@ -1051,7 +1066,9 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
TraceUtil.beginSection("createCodec:" + codecName); TraceUtil.beginSection("createCodec:" + codecName);
MediaCodec codec = MediaCodec.createByCodecName(codecName); MediaCodec codec = MediaCodec.createByCodecName(codecName);
if (enableAsynchronousBufferQueueing && Util.SDK_INT >= 23) { if (enableAsynchronousBufferQueueing && Util.SDK_INT >= 23) {
codecAdapter = new AsynchronousMediaCodecAdapter(codec, getTrackType()); codecAdapter =
new AsynchronousMediaCodecAdapter(
codec, getTrackType(), enableSynchronizeCodecInteractionsWithQueueing);
} else { } else {
codecAdapter = new SynchronousMediaCodecAdapter(codec); codecAdapter = new SynchronousMediaCodecAdapter(codec);
} }
......
library/core/src/test/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecAdapterTest.java
...@@ -46,7 +46,12 @@ public class AsynchronousMediaCodecAdapterTest { ...@@ -46,7 +46,12 @@ public class AsynchronousMediaCodecAdapterTest {
codec = MediaCodec.createByCodecName("h264"); codec = MediaCodec.createByCodecName("h264");
callbackThread = new HandlerThread("TestCallbackThread"); callbackThread = new HandlerThread("TestCallbackThread");
queueingThread = new HandlerThread("TestQueueingThread"); queueingThread = new HandlerThread("TestQueueingThread");
adapter = new AsynchronousMediaCodecAdapter(codec, callbackThread, queueingThread); adapter =
new AsynchronousMediaCodecAdapter(
codec,
callbackThread,
queueingThread,
/* synchronizeCodecInteractionsWithQueueing= */ false);
bufferInfo = new MediaCodec.BufferInfo(); bufferInfo = new MediaCodec.BufferInfo();
} }
......
library/core/src/test/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecBufferEnqueuerTest.java
...@@ -219,6 +219,28 @@ public class AsynchronousMediaCodecBufferEnqueuerTest { ...@@ -219,6 +219,28 @@ public class AsynchronousMediaCodecBufferEnqueuerTest {
assertThrows(IllegalStateException.class, () -> enqueuer.shutdown()); assertThrows(IllegalStateException.class, () -> enqueuer.shutdown());
} }
private static CryptoInfo createCryptoInfo() {
CryptoInfo info = new CryptoInfo();
int numSubSamples = 5;
int[] numBytesOfClearData = new int[] {0, 1, 2, 3};
int[] numBytesOfEncryptedData = new int[] {4, 5, 6, 7};
byte[] key = new byte[] {0, 1, 2, 3};
byte[] iv = new byte[] {4, 5, 6, 7};
@C.CryptoMode int mode = C.CRYPTO_MODE_AES_CBC;
int encryptedBlocks = 16;
int clearBlocks = 8;
info.set(
numSubSamples,
numBytesOfClearData,
numBytesOfEncryptedData,
key,
iv,
mode,
encryptedBlocks,
clearBlocks);
return info;
}
private static class TestHandlerThread extends HandlerThread { private static class TestHandlerThread extends HandlerThread {
private boolean started; private boolean started;
private boolean quit; private boolean quit;
...@@ -247,26 +269,4 @@ public class AsynchronousMediaCodecBufferEnqueuerTest { ...@@ -247,26 +269,4 @@ public class AsynchronousMediaCodecBufferEnqueuerTest {
return super.quit(); return super.quit();
} }
} }
private static CryptoInfo createCryptoInfo() {
CryptoInfo info = new CryptoInfo();
int numSubSamples = 5;
int[] numBytesOfClearData = new int[] {0, 1, 2, 3};
int[] numBytesOfEncryptedData = new int[] {4, 5, 6, 7};
byte[] key = new byte[] {0, 1, 2, 3};
byte[] iv = new byte[] {4, 5, 6, 7};
@C.CryptoMode int mode = C.CRYPTO_MODE_AES_CBC;
int encryptedBlocks = 16;
int clearBlocks = 8;
info.set(
numSubSamples,
numBytesOfClearData,
numBytesOfEncryptedData,
key,
iv,
mode,
encryptedBlocks,
clearBlocks);
return info;
}
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment