Skip to content

SDK / exoplayer

Go to a project
Commit 485949b5 by christosts Committed by Oliver Woodman
Options

Refactor AsynchronousMediaCoderAdapter 

Refactor the AsynchronousMediaCoderAdapter and move the callback thread
out of the adapter so that implementation of async callback and and
async queueing are consistent design-wise.

PiperOrigin-RevId: 338637837
parent 4783c329
Expand all
Inline Side-by-side
Showing with 376 additions and 398 deletions
library/core/src/main/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecAdapter.java
......@@ -19,32 +19,25 @@ package com.google.android.exoplayer2.mediacodec;
import android.media.MediaCodec;
import android.media.MediaCrypto;
import android.media.MediaFormat;
import android.os.Handler;
import android.os.HandlerThread;
import android.view.Surface;
import androidx.annotation.GuardedBy;
import androidx.annotation.IntDef;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import androidx.annotation.VisibleForTesting;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.decoder.CryptoInfo;
import com.google.android.exoplayer2.util.Util;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* A {@link MediaCodecAdapter} that operates the underlying {@link MediaCodec} in asynchronous mode
* and routes {@link MediaCodec.Callback} callbacks on a dedicated thread that is managed
* internally.
*
* <p>This adapter supports queueing input buffers asynchronously.
* A {@link MediaCodecAdapter} that operates the underlying {@link MediaCodec} in asynchronous mode,
* routes {@link MediaCodec.Callback} callbacks on a dedicated thread that is managed internally,
* and queues input buffers asynchronously.
*/
@RequiresApi(23)
/* package */ final class AsynchronousMediaCodecAdapter extends MediaCodec.Callback
implements MediaCodecAdapter {
/* package */ final class AsynchronousMediaCodecAdapter implements MediaCodecAdapter {
@Documented
@Retention(RetentionPolicy.SOURCE)
......@@ -56,24 +49,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private static final int STATE_STARTED = 2;
private static final int STATE_SHUT_DOWN = 3;
private final Object lock;
@GuardedBy("lock")
private final MediaCodecAsyncCallback mediaCodecAsyncCallback;
private final MediaCodec codec;
private final HandlerThread handlerThread;
private @MonotonicNonNull Handler handler;
@GuardedBy("lock")
private long pendingFlushCount;
private @State int state;
private final AsynchronousMediaCodecCallback asynchronousMediaCodecCallback;
private final AsynchronousMediaCodecBufferEnqueuer bufferEnqueuer;
@GuardedBy("lock")
@Nullable
private IllegalStateException internalException;
@State private int state;
/**
* Creates an instance that wraps the specified {@link MediaCodec}.
......@@ -85,21 +64,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/* package */ AsynchronousMediaCodecAdapter(MediaCodec codec, int trackType) {
this(
codec,
trackType,
new HandlerThread(createCallbackThreadLabel(trackType)),
new HandlerThread(createQueueingThreadLabel(trackType)));
}
@VisibleForTesting
/* package */ AsynchronousMediaCodecAdapter(
MediaCodec codec,
int trackType,
HandlerThread callbackThread,
HandlerThread enqueueingThread) {
this.lock = new Object();
this.mediaCodecAsyncCallback = new MediaCodecAsyncCallback();
MediaCodec codec, HandlerThread callbackThread, HandlerThread enqueueingThread) {
this.codec = codec;
this.handlerThread = callbackThread;
this.asynchronousMediaCodecCallback = new AsynchronousMediaCodecCallback(callbackThread);
this.bufferEnqueuer = new AsynchronousMediaCodecBufferEnqueuer(codec, enqueueingThread);
this.state = STATE_CREATED;
}
......@@ -110,9 +83,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@Nullable Surface surface,
@Nullable MediaCrypto crypto,
int flags) {
handlerThread.start();
handler = new Handler(handlerThread.getLooper());
codec.setCallback(this, handler);
asynchronousMediaCodecCallback.initialize(codec);
codec.configure(mediaFormat, surface, crypto, flags);
state = STATE_CONFIGURED;
}
......@@ -138,60 +109,40 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@Override
public int dequeueInputBufferIndex() {
synchronized (lock) {
if (isFlushing()) {
return MediaCodec.INFO_TRY_AGAIN_LATER;
} else {
maybeThrowException();
return mediaCodecAsyncCallback.dequeueInputBufferIndex();
}
}
return asynchronousMediaCodecCallback.dequeueInputBufferIndex();
}
@Override
public int dequeueOutputBufferIndex(MediaCodec.BufferInfo bufferInfo) {
synchronized (lock) {
if (isFlushing()) {
return MediaCodec.INFO_TRY_AGAIN_LATER;
} else {
maybeThrowException();
return mediaCodecAsyncCallback.dequeueOutputBufferIndex(bufferInfo);
}
}
return asynchronousMediaCodecCallback.dequeueOutputBufferIndex(bufferInfo);
}
@Override
public MediaFormat getOutputFormat() {
synchronized (lock) {
return mediaCodecAsyncCallback.getOutputFormat();
}
return asynchronousMediaCodecCallback.getOutputFormat();
}
@Override
public void flush() {
synchronized (lock) {
bufferEnqueuer.flush();
codec.flush();
++pendingFlushCount;
Util.castNonNull(handler).post(this::onFlushCompleted);
}
// The order of calls is important:
// First, flush the bufferEnqueuer to stop queueing input buffers.
// Second, flush the codec to stop producing available input/output buffers.
// Third, flush the callback after flushing the codec so that in-flight callbacks are discarded.
bufferEnqueuer.flush();
codec.flush();
// When flushAsync() is completed, start the codec again.
asynchronousMediaCodecCallback.flushAsync(/* onFlushCompleted= */ codec::start);
}
@Override
public void shutdown() {
synchronized (lock) {
if (state == STATE_STARTED) {
bufferEnqueuer.shutdown();
}
if (state == STATE_CONFIGURED || state == STATE_STARTED) {
handlerThread.quit();
mediaCodecAsyncCallback.flush();
// Leave the adapter in a flushing state so that
// it will not dequeue anything.
++pendingFlushCount;
asynchronousMediaCodecCallback.shutdown();
}
state = STATE_SHUT_DOWN;
}
}
@Override
......@@ -199,86 +150,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
return codec;
}
// Called from the handler thread.
@Override
public void onInputBufferAvailable(MediaCodec codec, int index) {
synchronized (lock) {
mediaCodecAsyncCallback.onInputBufferAvailable(codec, index);
}
}
@Override
public void onOutputBufferAvailable(MediaCodec codec, int index, MediaCodec.BufferInfo info) {
synchronized (lock) {
mediaCodecAsyncCallback.onOutputBufferAvailable(codec, index, info);
}
}
@Override
public void onError(MediaCodec codec, MediaCodec.CodecException e) {
synchronized (lock) {
mediaCodecAsyncCallback.onError(codec, e);
}
}
@Override
public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) {
synchronized (lock) {
mediaCodecAsyncCallback.onOutputFormatChanged(codec, format);
}
}
private void onFlushCompleted() {
synchronized (lock) {
onFlushCompletedSynchronized();
}
}
@GuardedBy("lock")
private void onFlushCompletedSynchronized() {
if (state == STATE_SHUT_DOWN) {
return;
}
--pendingFlushCount;
if (pendingFlushCount > 0) {
// Another flush() has been called.
return;
} else if (pendingFlushCount < 0) {
// This should never happen.
internalException = new IllegalStateException();
return;
}
mediaCodecAsyncCallback.flush();
try {
codec.start();
} catch (IllegalStateException e) {
internalException = e;
} catch (Exception e) {
internalException = new IllegalStateException(e);
}
}
@GuardedBy("lock")
private boolean isFlushing() {
return pendingFlushCount > 0;
}
@GuardedBy("lock")
private void maybeThrowException() {
maybeThrowInternalException();
mediaCodecAsyncCallback.maybeThrowMediaCodecException();
@VisibleForTesting
/* package */ void onError(MediaCodec.CodecException error) {
asynchronousMediaCodecCallback.onError(codec, error);
}
@GuardedBy("lock")
private void maybeThrowInternalException() {
if (internalException != null) {
IllegalStateException e = internalException;
internalException = null;
throw e;
}
@VisibleForTesting
/* package */ void onOutputFormatChanged(MediaFormat format) {
asynchronousMediaCodecCallback.onOutputFormatChanged(codec, format);
}
private static String createCallbackThreadLabel(int trackType) {
......
library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecAsyncCallback.java deleted 100644 → 0
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.mediacodec;
import android.media.MediaCodec;
import android.media.MediaFormat;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import androidx.annotation.VisibleForTesting;
import com.google.android.exoplayer2.util.IntArrayQueue;
import java.util.ArrayDeque;
/** Handles the asynchronous callbacks from {@link android.media.MediaCodec.Callback}. */
@RequiresApi(21)
/* package */ final class MediaCodecAsyncCallback extends MediaCodec.Callback {
private final IntArrayQueue availableInputBuffers;
private final IntArrayQueue availableOutputBuffers;
private final ArrayDeque<MediaCodec.BufferInfo> bufferInfos;
private final ArrayDeque<MediaFormat> formats;
@Nullable private MediaFormat currentFormat;
@Nullable private MediaFormat pendingOutputFormat;
@Nullable private IllegalStateException mediaCodecException;
/** Creates a new MediaCodecAsyncCallback. */
public MediaCodecAsyncCallback() {
availableInputBuffers = new IntArrayQueue();
availableOutputBuffers = new IntArrayQueue();
bufferInfos = new ArrayDeque<>();
formats = new ArrayDeque<>();
}
/**
* Returns the next available input buffer index or {@link MediaCodec#INFO_TRY_AGAIN_LATER} if no
* such buffer exists.
*/
public int dequeueInputBufferIndex() {
return availableInputBuffers.isEmpty()
? MediaCodec.INFO_TRY_AGAIN_LATER
: availableInputBuffers.remove();
}
/**
* Returns the next available output buffer index. If the next available output is a MediaFormat
* change, it will return {@link MediaCodec#INFO_OUTPUT_FORMAT_CHANGED} and you should call {@link
* #getOutputFormat()} to get the format. If there is no available output, this method will return
* {@link MediaCodec#INFO_TRY_AGAIN_LATER}.
*/
public int dequeueOutputBufferIndex(MediaCodec.BufferInfo bufferInfo) {
if (availableOutputBuffers.isEmpty()) {
return MediaCodec.INFO_TRY_AGAIN_LATER;
} else {
int bufferIndex = availableOutputBuffers.remove();
if (bufferIndex >= 0) {
MediaCodec.BufferInfo nextBufferInfo = bufferInfos.remove();
bufferInfo.set(
nextBufferInfo.offset,
nextBufferInfo.size,
nextBufferInfo.presentationTimeUs,
nextBufferInfo.flags);
} else if (bufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
currentFormat = formats.remove();
}
return bufferIndex;
}
}
/**
* Returns the {@link MediaFormat} signalled by the underlying {@link MediaCodec}.
*
* <p>Call this <b>after</b> {@link #dequeueOutputBufferIndex} returned {@link
* MediaCodec#INFO_OUTPUT_FORMAT_CHANGED}.
*
* @throws IllegalStateException If called before {@link #dequeueOutputBufferIndex} has returned
* {@link MediaCodec#INFO_OUTPUT_FORMAT_CHANGED}.
*/
public MediaFormat getOutputFormat() throws IllegalStateException {
if (currentFormat == null) {
throw new IllegalStateException();
}
return currentFormat;
}
/**
* Checks and throws an {@link IllegalStateException} if an error was previously set on this
* instance via {@link #onError}.
*/
public void maybeThrowMediaCodecException() throws IllegalStateException {
IllegalStateException exception = mediaCodecException;
mediaCodecException = null;
if (exception != null) {
throw exception;
}
}
/**
* Flushes the MediaCodecAsyncCallback. This method removes all available input and output buffers
* and any error that was previously set.
*/
public void flush() {
pendingOutputFormat = formats.isEmpty() ? null : formats.getLast();
availableInputBuffers.clear();
availableOutputBuffers.clear();
bufferInfos.clear();
formats.clear();
mediaCodecException = null;
}
@Override
public void onInputBufferAvailable(MediaCodec mediaCodec, int index) {
availableInputBuffers.add(index);
}
@Override
public void onOutputBufferAvailable(
MediaCodec mediaCodec, int index, MediaCodec.BufferInfo bufferInfo) {
if (pendingOutputFormat != null) {
addOutputFormat(pendingOutputFormat);
pendingOutputFormat = null;
}
availableOutputBuffers.add(index);
bufferInfos.add(bufferInfo);
}
@Override
public void onError(MediaCodec mediaCodec, MediaCodec.CodecException e) {
onMediaCodecError(e);
}
@Override
public void onOutputFormatChanged(MediaCodec mediaCodec, MediaFormat mediaFormat) {
addOutputFormat(mediaFormat);
pendingOutputFormat = null;
}
@VisibleForTesting()
void onMediaCodecError(IllegalStateException e) {
mediaCodecException = e;
}
private void addOutputFormat(MediaFormat mediaFormat) {
availableOutputBuffers.add(MediaCodec.INFO_OUTPUT_FORMAT_CHANGED);
formats.add(mediaFormat);
}
}
library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java
......@@ -1067,7 +1067,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
codecOperatingRate = CODEC_OPERATING_RATE_UNSET;
}
MediaCodecAdapter codecAdapter = null;
@Nullable MediaCodecAdapter codecAdapter = null;
try {
codecInitializingTimestamp = SystemClock.elapsedRealtime();
TraceUtil.beginSection("createCodec:" + codecName);
......
library/core/src/test/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecAdapterTest.java
......@@ -24,7 +24,6 @@ import android.media.MediaCodec;
import android.media.MediaFormat;
import android.os.HandlerThread;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.android.exoplayer2.C;
import java.io.IOException;
import java.lang.reflect.Constructor;
import org.junit.After;
......@@ -38,18 +37,16 @@ import org.robolectric.shadows.ShadowLooper;
public class AsynchronousMediaCodecAdapterTest {
private AsynchronousMediaCodecAdapter adapter;
private MediaCodec codec;
private TestHandlerThread callbackThread;
private HandlerThread callbackThread;
private HandlerThread queueingThread;
private MediaCodec.BufferInfo bufferInfo;
@Before
public void setUp() throws IOException {
codec = MediaCodec.createByCodecName("h264");
callbackThread = new TestHandlerThread("TestCallbackThread");
callbackThread = new HandlerThread("TestCallbackThread");
queueingThread = new HandlerThread("TestQueueingThread");
adapter =
new AsynchronousMediaCodecAdapter(
codec, /* trackType= */ C.TRACK_TYPE_VIDEO, callbackThread, queueingThread);
adapter = new AsynchronousMediaCodecAdapter(codec, callbackThread, queueingThread);
bufferInfo = new MediaCodec.BufferInfo();
}
......@@ -57,8 +54,6 @@ public class AsynchronousMediaCodecAdapterTest {
public void tearDown() {
adapter.shutdown();
codec.release();
assertThat(callbackThread.hasQuit()).isTrue();
}
@Test
......@@ -85,39 +80,7 @@ public class AsynchronousMediaCodecAdapterTest {
assertThat(adapter.dequeueInputBufferIndex()).isEqualTo(0);
}
@Test
public void dequeueInputBufferIndex_withPendingFlush_returnsTryAgainLater() {
adapter.configure(
createMediaFormat("foo"), /* surface= */ null, /* crypto= */ null, /* flags= */ 0);
adapter.start();
// After adapter.start(), the ShadowMediaCodec offers input buffer 0. We run all currently
// enqueued messages and pause the looper so that flush is not completed.
ShadowLooper shadowLooper = shadowOf(callbackThread.getLooper());
shadowLooper.idle();
shadowLooper.pause();
adapter.flush();
assertThat(adapter.dequeueInputBufferIndex()).isEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER);
}
@Test
public void dequeueInputBufferIndex_withFlushCompletedAndInputBuffer_returnsInputBuffer() {
adapter.configure(
createMediaFormat("foo"), /* surface= */ null, /* crypto= */ null, /* flags= */ 0);
adapter.start();
// After adapter.start(), the ShadowMediaCodec offers input buffer 0. We advance the looper to
// make sure all messages have been propagated to the adapter.
ShadowLooper shadowLooper = shadowOf(callbackThread.getLooper());
shadowLooper.idle();
adapter.flush();
// Progress the looper to complete flush(): the adapter should call codec.start(), triggering
// the ShadowMediaCodec to offer input buffer 0.
shadowLooper.idle();
assertThat(adapter.dequeueInputBufferIndex()).isEqualTo(0);
}
@Test
public void dequeueInputBufferIndex_withMediaCodecError_throwsException() throws Exception {
......@@ -128,7 +91,7 @@ public class AsynchronousMediaCodecAdapterTest {
adapter.start();
// Set an error directly on the adapter (not through the looper).
adapter.onError(codec, createCodecException());
adapter.onError(createCodecException());
assertThrows(IllegalStateException.class, () -> adapter.dequeueInputBufferIndex());
}
......@@ -193,25 +156,6 @@ public class AsynchronousMediaCodecAdapterTest {
}
@Test
public void dequeueOutputBufferIndex_withPendingFlush_returnsTryAgainLater() {
adapter.configure(
createMediaFormat("foo"), /* surface= */ null, /* crypto= */ null, /* flags= */ 0);
adapter.start();
// After start(), the ShadowMediaCodec offers input buffer 0, which is available only if we
// progress the adapter's looper.
ShadowLooper shadowLooper = shadowOf(callbackThread.getLooper());
shadowLooper.idle();
// Flush enqueues a task in the looper, but we will pause the looper to leave flush()
// in an incomplete state.
shadowLooper.pause();
adapter.flush();
assertThat(adapter.dequeueOutputBufferIndex(bufferInfo))
.isEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER);
}
@Test
public void dequeueOutputBufferIndex_withMediaCodecError_throwsException() throws Exception {
// Pause the looper so that we interact with the adapter from this thread only.
adapter.configure(
......@@ -220,7 +164,7 @@ public class AsynchronousMediaCodecAdapterTest {
adapter.start();
// Set an error directly on the adapter.
adapter.onError(codec, createCodecException());
adapter.onError(createCodecException());
assertThrows(IllegalStateException.class, () -> adapter.dequeueOutputBufferIndex(bufferInfo));
}
......@@ -266,8 +210,8 @@ public class AsynchronousMediaCodecAdapterTest {
// progress the adapter's looper.
shadowOf(callbackThread.getLooper()).idle();
// Add another format directly on the adapter.
adapter.onOutputFormatChanged(codec, createMediaFormat("format2"));
// Add another format on the adapter.
adapter.onOutputFormatChanged(createMediaFormat("format2"));
assertThat(adapter.dequeueOutputBufferIndex(bufferInfo))
.isEqualTo(MediaCodec.INFO_OUTPUT_FORMAT_CHANGED);
......@@ -314,22 +258,4 @@ public class AsynchronousMediaCodecAdapterTest {
return constructor.newInstance(
/* errorCode= */ 0, /* actionCode= */ 0, /* detailMessage= */ "error from codec");
}
private static class TestHandlerThread extends HandlerThread {
private boolean quit;
TestHandlerThread(String label) {
super(label);
}
public boolean hasQuit() {
return quit;
}
@Override
public boolean quit() {
quit = true;
return super.quit();
}
}
}
library/core/src/test/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecBufferEnqueuerTest.java
......@@ -30,7 +30,6 @@ import com.google.android.exoplayer2.decoder.CryptoInfo;
import com.google.android.exoplayer2.util.ConditionVariable;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.concurrent.atomic.AtomicLong;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
......@@ -65,7 +64,7 @@ public class AsynchronousMediaCodecBufferEnqueuerTest {
enqueuer.shutdown();
codec.stop();
codec.release();
assertThat(TestHandlerThread.INSTANCES_STARTED.get()).isEqualTo(0);
assertThat(!handlerThread.hasStarted() || handlerThread.hasQuit()).isTrue();
}
@Test
......@@ -221,25 +220,31 @@ public class AsynchronousMediaCodecBufferEnqueuerTest {
}
private static class TestHandlerThread extends HandlerThread {
private static final AtomicLong INSTANCES_STARTED = new AtomicLong(0);
private boolean started;
private boolean quit;
TestHandlerThread(String name) {
super(name);
TestHandlerThread(String label) {
super(label);
}
public boolean hasStarted() {
return started;
}
public boolean hasQuit() {
return quit;
}
@Override
public synchronized void start() {
super.start();
INSTANCES_STARTED.incrementAndGet();
started = true;
}
@Override
public boolean quit() {
boolean quit = super.quit();
if (quit) {
INSTANCES_STARTED.decrementAndGet();
}
return quit;
quit = true;
return super.quit();
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment