Commit 5725acb7 by christosts Committed by kim-vde

Defer MediaCodec queueing to background thread

The DedicatedThreadAsyncMediaCodecAdapter supports enqueueing
input buffers in a background Thread.

PiperOrigin-RevId: 294202744
parent 278e1aff
...@@ -15,9 +15,11 @@ ...@@ -15,9 +15,11 @@
*/ */
package com.google.android.exoplayer2.decoder; package com.google.android.exoplayer2.decoder;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi; import androidx.annotation.RequiresApi;
import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.util.Util;
import java.util.Arrays;
/** /**
* Compatibility wrapper for {@link android.media.MediaCodec.CryptoInfo}. * Compatibility wrapper for {@link android.media.MediaCodec.CryptoInfo}.
...@@ -105,12 +107,76 @@ public final class CryptoInfo { ...@@ -105,12 +107,76 @@ public final class CryptoInfo {
return frameworkCryptoInfo; return frameworkCryptoInfo;
} }
/** Performs a deep copy to {@code cryptoInfo}. */
public void copyTo(android.media.MediaCodec.CryptoInfo cryptoInfo) {
// Update cryptoInfo fields directly because CryptoInfo.set performs an unnecessary
// object allocation on Android N.
cryptoInfo.numSubSamples = numSubSamples;
cryptoInfo.numBytesOfClearData = copyOrNull(frameworkCryptoInfo.numBytesOfClearData);
cryptoInfo.numBytesOfEncryptedData = copyOrNull(frameworkCryptoInfo.numBytesOfEncryptedData);
cryptoInfo.key = copyOrNull(frameworkCryptoInfo.key);
cryptoInfo.iv = copyOrNull(frameworkCryptoInfo.iv);
cryptoInfo.mode = mode;
if (Util.SDK_INT >= 24) {
android.media.MediaCodec.CryptoInfo.Pattern pattern = patternHolder.pattern;
android.media.MediaCodec.CryptoInfo.Pattern patternCopy =
new android.media.MediaCodec.CryptoInfo.Pattern(
pattern.getEncryptBlocks(), pattern.getSkipBlocks());
cryptoInfo.setPattern(patternCopy);
}
}
/** @deprecated Use {@link #getFrameworkCryptoInfo()}. */ /** @deprecated Use {@link #getFrameworkCryptoInfo()}. */
@Deprecated @Deprecated
public android.media.MediaCodec.CryptoInfo getFrameworkCryptoInfoV16() { public android.media.MediaCodec.CryptoInfo getFrameworkCryptoInfoV16() {
return getFrameworkCryptoInfo(); return getFrameworkCryptoInfo();
} }
/**
* Increases the number of clear data for the first sub sample by {@code count}.
*
* <p>If {@code count} is 0, this method is a no-op. Otherwise, it adds {@code count} to {@link
* #numBytesOfClearData}[0].
*
* <p>If {@link #numBytesOfClearData} is null (which is permitted), this method will instantiate
* it to a new {@code int[1]}.
*
* @param count The number of bytes to be added to the first subSample of {@link
* #numBytesOfClearData}.
*/
public void increaseClearDataFirstSubSampleBy(int count) {
if (count == 0) {
return;
}
if (numBytesOfClearData == null) {
numBytesOfClearData = new int[1];
}
numBytesOfClearData[0] += count;
// It is OK to have numBytesOfClearData and frameworkCryptoInfo.numBytesOfClearData point to
// the same array, see set().
if (frameworkCryptoInfo.numBytesOfClearData == null) {
frameworkCryptoInfo.numBytesOfClearData = numBytesOfClearData;
}
// Update frameworkCryptoInfo.numBytesOfClearData only if it points to a different array than
// numBytesOfClearData (all fields are public and non-final, therefore they can set be set
// directly without calling set()). Otherwise, the array has been updated already in the steps
// above.
if (frameworkCryptoInfo.numBytesOfClearData != numBytesOfClearData) {
frameworkCryptoInfo.numBytesOfClearData[0] += count;
}
}
private static int[] copyOrNull(@Nullable int[] array) {
return array != null ? Arrays.copyOf(array, array.length) : null;
}
private static byte[] copyOrNull(@Nullable byte[] array) {
return array != null ? Arrays.copyOf(array, array.length) : null;
}
@RequiresApi(24) @RequiresApi(24)
private static final class PatternHolderV24 { private static final class PatternHolderV24 {
......
/*
* Copyright (C) 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.decoder;
import static com.google.common.truth.Truth.assertThat;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
/** Unit tests for {@link CryptoInfo} */
@RunWith(AndroidJUnit4.class)
public class CryptoInfoTest {
private CryptoInfo cryptoInfo;
@Before
public void setUp() {
cryptoInfo = new CryptoInfo();
}
@Test
public void increaseClearDataFirstSubSampleBy_numBytesOfClearDataIsNullAndZeroInput_isNoOp() {
cryptoInfo.increaseClearDataFirstSubSampleBy(0);
assertThat(cryptoInfo.numBytesOfClearData).isNull();
assertThat(cryptoInfo.getFrameworkCryptoInfo().numBytesOfClearData).isNull();
}
@Test
public void increaseClearDataFirstSubSampleBy_withNumBytesOfClearDataSetAndZeroInput_isNoOp() {
int[] data = new int[] {1, 1, 1, 1};
cryptoInfo.numBytesOfClearData = data;
cryptoInfo.getFrameworkCryptoInfo().numBytesOfClearData = data;
cryptoInfo.increaseClearDataFirstSubSampleBy(5);
assertThat(cryptoInfo.numBytesOfClearData[0]).isEqualTo(6);
assertThat(cryptoInfo.getFrameworkCryptoInfo().numBytesOfClearData[0]).isEqualTo(6);
}
@Test
public void increaseClearDataFirstSubSampleBy_withSharedClearDataPointer_setsValue() {
int[] data = new int[] {1, 1, 1, 1};
cryptoInfo.numBytesOfClearData = data;
cryptoInfo.getFrameworkCryptoInfo().numBytesOfClearData = data;
cryptoInfo.increaseClearDataFirstSubSampleBy(5);
assertThat(cryptoInfo.numBytesOfClearData[0]).isEqualTo(6);
assertThat(cryptoInfo.getFrameworkCryptoInfo().numBytesOfClearData[0]).isEqualTo(6);
}
@Test
public void increaseClearDataFirstSubSampleBy_withDifferentClearDataArrays_setsValue() {
cryptoInfo.numBytesOfClearData = new int[] {1, 1, 1, 1};
cryptoInfo.getFrameworkCryptoInfo().numBytesOfClearData = new int[] {5, 5, 5, 5};
cryptoInfo.increaseClearDataFirstSubSampleBy(5);
assertThat(cryptoInfo.numBytesOfClearData[0]).isEqualTo(6);
assertThat(cryptoInfo.getFrameworkCryptoInfo().numBytesOfClearData[0]).isEqualTo(10);
}
@Test
public void increaseClearDataFirstSubSampleBy_withInternalClearDataArraysNull_setsValue() {
cryptoInfo.numBytesOfClearData = new int[] {10, 10, 10, 10};
cryptoInfo.increaseClearDataFirstSubSampleBy(5);
assertThat(cryptoInfo.numBytesOfClearData[0]).isEqualTo(15);
assertThat(cryptoInfo.getFrameworkCryptoInfo().numBytesOfClearData[0]).isEqualTo(15);
}
@Test
public void increaseClearDataFirstSubSampleBy_internalClearDataIsNotNull_setsValue() {
cryptoInfo.getFrameworkCryptoInfo().numBytesOfClearData = new int[] {5, 5, 5, 5};
cryptoInfo.increaseClearDataFirstSubSampleBy(5);
assertThat(cryptoInfo.numBytesOfClearData[0]).isEqualTo(5);
assertThat(cryptoInfo.getFrameworkCryptoInfo().numBytesOfClearData[0]).isEqualTo(10);
}
}
...@@ -23,6 +23,7 @@ import android.os.Looper; ...@@ -23,6 +23,7 @@ import android.os.Looper;
import androidx.annotation.Nullable; import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi; import androidx.annotation.RequiresApi;
import androidx.annotation.VisibleForTesting; import androidx.annotation.VisibleForTesting;
import com.google.android.exoplayer2.decoder.CryptoInfo;
import com.google.android.exoplayer2.util.Assertions; import com.google.android.exoplayer2.util.Assertions;
/** /**
...@@ -71,8 +72,9 @@ import com.google.android.exoplayer2.util.Assertions; ...@@ -71,8 +72,9 @@ import com.google.android.exoplayer2.util.Assertions;
@Override @Override
public void queueSecureInputBuffer( public void queueSecureInputBuffer(
int index, int offset, MediaCodec.CryptoInfo info, long presentationTimeUs, int flags) { int index, int offset, CryptoInfo info, long presentationTimeUs, int flags) {
codec.queueSecureInputBuffer(index, offset, info, presentationTimeUs, flags); codec.queueSecureInputBuffer(
index, offset, info.getFrameworkCryptoInfo(), presentationTimeUs, flags);
} }
@Override @Override
......
/*
* Copyright (C) 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.mediacodec;
import android.media.MediaCodec;
import android.os.Handler;
import android.os.HandlerThread;
import android.os.Message;
import androidx.annotation.GuardedBy;
import androidx.annotation.NonNull;
import androidx.annotation.RequiresApi;
import androidx.annotation.VisibleForTesting;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.decoder.CryptoInfo;
import com.google.android.exoplayer2.util.ConditionVariable;
import com.google.android.exoplayer2.util.Util;
import java.util.ArrayDeque;
import java.util.concurrent.atomic.AtomicReference;
import org.checkerframework.checker.nullness.compatqual.NullableType;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* A {@link MediaCodecInputBufferEnqueuer} that defers queueing operations on a background thread.
*
* <p>The implementation of this class assumes that its public methods will be called from the same
* thread.
*/
@RequiresApi(23)
class AsynchronousMediaCodecBufferEnqueuer implements MediaCodecInputBufferEnqueuer {
private static final int MSG_QUEUE_INPUT_BUFFER = 0;
private static final int MSG_QUEUE_SECURE_INPUT_BUFFER = 1;
private static final int MSG_FLUSH = 2;
@GuardedBy("MESSAGE_PARAMS_INSTANCE_POOL")
private static final ArrayDeque<MessageParams> MESSAGE_PARAMS_INSTANCE_POOL = new ArrayDeque<>();
private final MediaCodec codec;
private final HandlerThread handlerThread;
private @MonotonicNonNull Handler handler;
private final AtomicReference<@NullableType RuntimeException> pendingRuntimeException;
private final ConditionVariable conditionVariable;
private boolean started;
/**
* Creates a new instance that submits input buffers on the specified {@link MediaCodec}.
*
* @param codec The {@link MediaCodec} to submit input buffers to.
* @param trackType The type of stream (used for debug logs).
*/
public AsynchronousMediaCodecBufferEnqueuer(MediaCodec codec, int trackType) {
this(
codec,
new HandlerThread(createThreadLabel(trackType)),
/* conditionVariable= */ new ConditionVariable());
}
@VisibleForTesting
/* package */ AsynchronousMediaCodecBufferEnqueuer(
MediaCodec codec, HandlerThread handlerThread, ConditionVariable conditionVariable) {
this.codec = codec;
this.handlerThread = handlerThread;
this.conditionVariable = conditionVariable;
pendingRuntimeException = new AtomicReference<>();
}
@Override
public void start() {
if (!started) {
handlerThread.start();
handler =
new Handler(handlerThread.getLooper()) {
@Override
public void handleMessage(@NonNull Message msg) {
doHandleMessage(msg);
}
};
started = true;
}
}
@Override
public void queueInputBuffer(
int index, int offset, int size, long presentationTimeUs, int flags) {
maybeThrowException();
MessageParams messageParams = getMessageParams();
messageParams.setQueueParams(index, offset, size, presentationTimeUs, flags);
Message message =
Util.castNonNull(handler).obtainMessage(MSG_QUEUE_INPUT_BUFFER, messageParams);
message.sendToTarget();
}
@Override
public void queueSecureInputBuffer(
int index, int offset, CryptoInfo info, long presentationTimeUs, int flags) {
maybeThrowException();
MessageParams messageParams = getMessageParams();
messageParams.setQueueParams(index, offset, /* size= */ 0, presentationTimeUs, flags);
info.copyTo(messageParams.cryptoInfo);
Message message =
Util.castNonNull(handler).obtainMessage(MSG_QUEUE_SECURE_INPUT_BUFFER, messageParams);
message.sendToTarget();
}
@Override
public void flush() {
if (started) {
try {
flushHandlerThread();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
// The playback thread should not be interrupted. Raising this as an
// IllegalStateException.
throw new IllegalStateException(e);
}
}
}
@Override
public void shutdown() {
if (started) {
flush();
handlerThread.quit();
}
started = false;
}
private void doHandleMessage(Message msg) {
MessageParams params = null;
switch (msg.what) {
case MSG_QUEUE_INPUT_BUFFER:
params = (MessageParams) msg.obj;
doQueueInputBuffer(
params.index, params.offset, params.size, params.presentationTimeUs, params.flags);
break;
case MSG_QUEUE_SECURE_INPUT_BUFFER:
params = (MessageParams) msg.obj;
doQueueSecureInputBuffer(
params.index,
params.offset,
params.cryptoInfo,
params.presentationTimeUs,
params.flags);
break;
case MSG_FLUSH:
conditionVariable.open();
break;
default:
setPendingRuntimeException(new IllegalStateException(String.valueOf(msg.what)));
}
if (params != null) {
recycleMessageParams(params);
}
}
private void maybeThrowException() {
RuntimeException exception = pendingRuntimeException.getAndSet(null);
if (exception != null) {
throw exception;
}
}
/**
* Empties all tasks enqueued on the {@link #handlerThread} via the {@link #handler}. This method
* blocks until the {@link #handlerThread} is idle.
*/
private void flushHandlerThread() throws InterruptedException {
Handler handler = Util.castNonNull(this.handler);
handler.removeCallbacksAndMessages(null);
conditionVariable.close();
handler.obtainMessage(MSG_FLUSH).sendToTarget();
conditionVariable.block();
// Check if any exceptions happened during the last queueing action.
maybeThrowException();
}
// Called from the handler thread
@VisibleForTesting
/* package */ void setPendingRuntimeException(RuntimeException exception) {
pendingRuntimeException.set(exception);
}
private void doQueueInputBuffer(
int index, int offset, int size, long presentationTimeUs, int flag) {
try {
codec.queueInputBuffer(index, offset, size, presentationTimeUs, flag);
} catch (RuntimeException e) {
setPendingRuntimeException(e);
}
}
private void doQueueSecureInputBuffer(
int index, int offset, MediaCodec.CryptoInfo info, long presentationTimeUs, int flags) {
try {
codec.queueSecureInputBuffer(index, offset, info, presentationTimeUs, flags);
} catch (RuntimeException e) {
setPendingRuntimeException(e);
}
}
@VisibleForTesting
/* package */ static int getInstancePoolSize() {
synchronized (MESSAGE_PARAMS_INSTANCE_POOL) {
return MESSAGE_PARAMS_INSTANCE_POOL.size();
}
}
private static MessageParams getMessageParams() {
synchronized (MESSAGE_PARAMS_INSTANCE_POOL) {
if (MESSAGE_PARAMS_INSTANCE_POOL.isEmpty()) {
return new MessageParams();
} else {
return MESSAGE_PARAMS_INSTANCE_POOL.removeFirst();
}
}
}
private static void recycleMessageParams(MessageParams params) {
synchronized (MESSAGE_PARAMS_INSTANCE_POOL) {
MESSAGE_PARAMS_INSTANCE_POOL.add(params);
}
}
/** Parameters for queue input buffer and queue secure input buffer tasks. */
private static class MessageParams {
public int index;
public int offset;
public int size;
public final MediaCodec.CryptoInfo cryptoInfo;
public long presentationTimeUs;
public int flags;
MessageParams() {
cryptoInfo = new MediaCodec.CryptoInfo();
}
/** Convenience method for setting the queueing parameters. */
public void setQueueParams(
int index, int offset, int size, long presentationTimeUs, int flags) {
this.index = index;
this.offset = offset;
this.size = size;
this.presentationTimeUs = presentationTimeUs;
this.flags = flags;
}
}
private static String createThreadLabel(int trackType) {
StringBuilder labelBuilder = new StringBuilder("MediaCodecInputBufferEnqueuer:");
if (trackType == C.TRACK_TYPE_AUDIO) {
labelBuilder.append("Audio");
} else if (trackType == C.TRACK_TYPE_VIDEO) {
labelBuilder.append("Video");
} else {
labelBuilder.append("Unknown(").append(trackType).append(")");
}
return labelBuilder.toString();
}
}
...@@ -26,6 +26,7 @@ import androidx.annotation.Nullable; ...@@ -26,6 +26,7 @@ import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi; import androidx.annotation.RequiresApi;
import androidx.annotation.VisibleForTesting; import androidx.annotation.VisibleForTesting;
import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.decoder.CryptoInfo;
import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.util.Util;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
...@@ -33,6 +34,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -33,6 +34,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* A {@link MediaCodecAdapter} that operates the underlying {@link MediaCodec} in asynchronous mode * A {@link MediaCodecAdapter} that operates the underlying {@link MediaCodec} in asynchronous mode
* and routes {@link MediaCodec.Callback} callbacks on a dedicated Thread that is managed * and routes {@link MediaCodec.Callback} callbacks on a dedicated Thread that is managed
* internally. * internally.
*
* <p>This adapter supports queueing input buffers asynchronously.
*/ */
@RequiresApi(23) @RequiresApi(23)
/* package */ final class DedicatedThreadAsyncMediaCodecAdapter extends MediaCodec.Callback /* package */ final class DedicatedThreadAsyncMediaCodecAdapter extends MediaCodec.Callback
...@@ -56,27 +59,56 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -56,27 +59,56 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@Nullable private IllegalStateException internalException; @Nullable private IllegalStateException internalException;
/** /**
* Creates an instance that wraps the specified {@link MediaCodec}. Instances created with this
* constructor will queue input buffers to the {@link MediaCodec} synchronously.
*
* @param codec The {@link MediaCodec} to wrap.
* @param trackType One of {@link C#TRACK_TYPE_AUDIO} or {@link C#TRACK_TYPE_VIDEO}. Used for
* labelling the internal Thread accordingly.
*/
/* package */ DedicatedThreadAsyncMediaCodecAdapter(MediaCodec codec, int trackType) {
this(
codec,
/* enableAsynchronousQueueing= */ false,
trackType,
new HandlerThread(createThreadLabel(trackType)));
}
/**
* Creates an instance that wraps the specified {@link MediaCodec}. * Creates an instance that wraps the specified {@link MediaCodec}.
* *
* @param codec The {@link MediaCodec} to wrap. * @param codec The {@link MediaCodec} to wrap.
* @param enableAsynchronousQueueing Whether input buffers will be queued asynchronously.
* @param trackType One of {@link C#TRACK_TYPE_AUDIO} or {@link C#TRACK_TYPE_VIDEO}. Used for * @param trackType One of {@link C#TRACK_TYPE_AUDIO} or {@link C#TRACK_TYPE_VIDEO}. Used for
* labelling the internal Thread accordingly. * labelling the internal Thread accordingly.
* @throws IllegalArgumentException If {@code trackType} is not one of {@link C#TRACK_TYPE_AUDIO} * @throws IllegalArgumentException If {@code trackType} is not one of {@link C#TRACK_TYPE_AUDIO}
* or {@link C#TRACK_TYPE_VIDEO}. * or {@link C#TRACK_TYPE_VIDEO}.
*/ */
/* package */ DedicatedThreadAsyncMediaCodecAdapter(MediaCodec codec, int trackType) { /* package */ DedicatedThreadAsyncMediaCodecAdapter(
this(codec, new HandlerThread(createThreadLabel(trackType))); MediaCodec codec, boolean enableAsynchronousQueueing, int trackType) {
this(
codec,
enableAsynchronousQueueing,
trackType,
new HandlerThread(createThreadLabel(trackType)));
} }
@VisibleForTesting @VisibleForTesting
/* package */ DedicatedThreadAsyncMediaCodecAdapter( /* package */ DedicatedThreadAsyncMediaCodecAdapter(
MediaCodec codec, HandlerThread handlerThread) { MediaCodec codec,
boolean enableAsynchronousQueueing,
int trackType,
HandlerThread handlerThread) {
mediaCodecAsyncCallback = new MediaCodecAsyncCallback(); mediaCodecAsyncCallback = new MediaCodecAsyncCallback();
this.codec = codec; this.codec = codec;
this.handlerThread = handlerThread; this.handlerThread = handlerThread;
state = STATE_CREATED; state = STATE_CREATED;
codecStartRunnable = codec::start; codecStartRunnable = codec::start;
bufferEnqueuer = new SynchronousMediaCodecBufferEnqueuer(this.codec); if (enableAsynchronousQueueing) {
bufferEnqueuer = new AsynchronousMediaCodecBufferEnqueuer(codec, trackType);
} else {
bufferEnqueuer = new SynchronousMediaCodecBufferEnqueuer(this.codec);
}
} }
@Override @Override
...@@ -99,7 +131,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -99,7 +131,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@Override @Override
public void queueSecureInputBuffer( public void queueSecureInputBuffer(
int index, int offset, MediaCodec.CryptoInfo info, long presentationTimeUs, int flags) { int index, int offset, CryptoInfo info, long presentationTimeUs, int flags) {
// This method does not need to be synchronized because it does not interact with the // This method does not need to be synchronized because it does not interact with the
// mediaCodecAsyncCallback. // mediaCodecAsyncCallback.
bufferEnqueuer.queueSecureInputBuffer(index, offset, info, presentationTimeUs, flags); bufferEnqueuer.queueSecureInputBuffer(index, offset, info, presentationTimeUs, flags);
......
...@@ -18,6 +18,7 @@ package com.google.android.exoplayer2.mediacodec; ...@@ -18,6 +18,7 @@ package com.google.android.exoplayer2.mediacodec;
import android.media.MediaCodec; import android.media.MediaCodec;
import android.media.MediaFormat; import android.media.MediaFormat;
import com.google.android.exoplayer2.decoder.CryptoInfo;
/** /**
* Abstracts {@link MediaCodec} operations. * Abstracts {@link MediaCodec} operations.
...@@ -81,10 +82,14 @@ import android.media.MediaFormat; ...@@ -81,10 +82,14 @@ import android.media.MediaFormat;
* <p>The {@code index} must be an input buffer index that has been obtained from a previous call * <p>The {@code index} must be an input buffer index that has been obtained from a previous call
* to {@link #dequeueInputBufferIndex()}. * to {@link #dequeueInputBufferIndex()}.
* *
* <p>Note: This method behaves as {@link MediaCodec#queueSecureInputBuffer} with the difference
* that {@code info} is of type {@link CryptoInfo} and not {@link
* android.media.MediaCodec.CryptoInfo}.
*
* @see MediaCodec#queueSecureInputBuffer * @see MediaCodec#queueSecureInputBuffer
*/ */
void queueSecureInputBuffer( void queueSecureInputBuffer(
int index, int offset, MediaCodec.CryptoInfo info, long presentationTimeUs, int flags); int index, int offset, CryptoInfo info, long presentationTimeUs, int flags);
/** /**
* Flushes the {@code MediaCodecAdapter}. * Flushes the {@code MediaCodecAdapter}.
......
...@@ -17,6 +17,7 @@ ...@@ -17,6 +17,7 @@
package com.google.android.exoplayer2.mediacodec; package com.google.android.exoplayer2.mediacodec;
import android.media.MediaCodec; import android.media.MediaCodec;
import com.google.android.exoplayer2.decoder.CryptoInfo;
/** Abstracts operations to enqueue input buffer on a {@link android.media.MediaCodec}. */ /** Abstracts operations to enqueue input buffer on a {@link android.media.MediaCodec}. */
interface MediaCodecInputBufferEnqueuer { interface MediaCodecInputBufferEnqueuer {
...@@ -38,10 +39,14 @@ interface MediaCodecInputBufferEnqueuer { ...@@ -38,10 +39,14 @@ interface MediaCodecInputBufferEnqueuer {
/** /**
* Submits an input buffer that potentially contains encrypted data for decoding. * Submits an input buffer that potentially contains encrypted data for decoding.
* *
* @see MediaCodec#queueSecureInputBuffer * <p>Note: This method behaves as {@link MediaCodec#queueSecureInputBuffer} with the difference
* that {@code info} is of type {@link CryptoInfo} and not {@link
* android.media.MediaCodec.CryptoInfo}.
*
* @see android.media.MediaCodec#queueSecureInputBuffer
*/ */
void queueSecureInputBuffer( void queueSecureInputBuffer(
int index, int offset, MediaCodec.CryptoInfo info, long presentationTimeUs, int flags); int index, int offset, CryptoInfo info, long presentationTimeUs, int flags);
/** Flushes the instance. */ /** Flushes the instance. */
void flush(); void flush();
......
...@@ -33,6 +33,7 @@ import com.google.android.exoplayer2.C; ...@@ -33,6 +33,7 @@ import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.ExoPlaybackException; import com.google.android.exoplayer2.ExoPlaybackException;
import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.FormatHolder; import com.google.android.exoplayer2.FormatHolder;
import com.google.android.exoplayer2.decoder.CryptoInfo;
import com.google.android.exoplayer2.decoder.DecoderCounters; import com.google.android.exoplayer2.decoder.DecoderCounters;
import com.google.android.exoplayer2.decoder.DecoderInputBuffer; import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
import com.google.android.exoplayer2.drm.DrmSession; import com.google.android.exoplayer2.drm.DrmSession;
...@@ -81,7 +82,8 @@ public abstract class MediaCodecRenderer extends BaseRenderer { ...@@ -81,7 +82,8 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
OPERATION_MODE_SYNCHRONOUS, OPERATION_MODE_SYNCHRONOUS,
OPERATION_MODE_ASYNCHRONOUS_PLAYBACK_THREAD, OPERATION_MODE_ASYNCHRONOUS_PLAYBACK_THREAD,
OPERATION_MODE_ASYNCHRONOUS_DEDICATED_THREAD, OPERATION_MODE_ASYNCHRONOUS_DEDICATED_THREAD,
OPERATION_MODE_ASYNCHRONOUS_DEDICATED_THREAD_MULTI_LOCK OPERATION_MODE_ASYNCHRONOUS_DEDICATED_THREAD_MULTI_LOCK,
OPERATION_MODE_ASYNCHRONOUS_QUEUEING
}) })
public @interface MediaCodecOperationMode {} public @interface MediaCodecOperationMode {}
...@@ -102,6 +104,11 @@ public abstract class MediaCodecRenderer extends BaseRenderer { ...@@ -102,6 +104,11 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
* callbacks to a dedicated Thread. Uses granular locking for input and output buffers. * callbacks to a dedicated Thread. Uses granular locking for input and output buffers.
*/ */
public static final int OPERATION_MODE_ASYNCHRONOUS_DEDICATED_THREAD_MULTI_LOCK = 3; public static final int OPERATION_MODE_ASYNCHRONOUS_DEDICATED_THREAD_MULTI_LOCK = 3;
/**
* Operates the {@link MediaCodec} in asynchronous mode, routes {@link MediaCodec.Callback}
* callbacks to a dedicated Thread, and offloads queueing to another Thread.
*/
public static final int OPERATION_MODE_ASYNCHRONOUS_QUEUEING = 4;
/** Thrown when a failure occurs instantiating a decoder. */ /** Thrown when a failure occurs instantiating a decoder. */
public static class DecoderInitializationException extends Exception { public static class DecoderInitializationException extends Exception {
...@@ -1020,6 +1027,11 @@ public abstract class MediaCodecRenderer extends BaseRenderer { ...@@ -1020,6 +1027,11 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
} else if (mediaCodecOperationMode == OPERATION_MODE_ASYNCHRONOUS_DEDICATED_THREAD_MULTI_LOCK } else if (mediaCodecOperationMode == OPERATION_MODE_ASYNCHRONOUS_DEDICATED_THREAD_MULTI_LOCK
&& Util.SDK_INT >= 23) { && Util.SDK_INT >= 23) {
codecAdapter = new MultiLockAsyncMediaCodecAdapter(codec, getTrackType()); codecAdapter = new MultiLockAsyncMediaCodecAdapter(codec, getTrackType());
} else if (mediaCodecOperationMode == OPERATION_MODE_ASYNCHRONOUS_QUEUEING
&& Util.SDK_INT >= 23) {
codecAdapter =
new DedicatedThreadAsyncMediaCodecAdapter(
codec, /* enableAsynchronousQueueing= */ true, getTrackType());
} else { } else {
codecAdapter = new SynchronousMediaCodecAdapter(codec); codecAdapter = new SynchronousMediaCodecAdapter(codec);
} }
...@@ -1275,8 +1287,8 @@ public abstract class MediaCodecRenderer extends BaseRenderer { ...@@ -1275,8 +1287,8 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
onQueueInputBuffer(buffer); onQueueInputBuffer(buffer);
if (bufferEncrypted) { if (bufferEncrypted) {
MediaCodec.CryptoInfo cryptoInfo = getFrameworkCryptoInfo(buffer, CryptoInfo cryptoInfo = buffer.cryptoInfo;
adaptiveReconfigurationBytes); cryptoInfo.increaseClearDataFirstSubSampleBy(adaptiveReconfigurationBytes);
codecAdapter.queueSecureInputBuffer(inputIndex, 0, cryptoInfo, presentationTimeUs, 0); codecAdapter.queueSecureInputBuffer(inputIndex, 0, cryptoInfo, presentationTimeUs, 0);
} else { } else {
codecAdapter.queueInputBuffer(inputIndex, 0, buffer.data.limit(), presentationTimeUs, 0); codecAdapter.queueInputBuffer(inputIndex, 0, buffer.data.limit(), presentationTimeUs, 0);
...@@ -1889,22 +1901,6 @@ public abstract class MediaCodecRenderer extends BaseRenderer { ...@@ -1889,22 +1901,6 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
} }
} }
private static MediaCodec.CryptoInfo getFrameworkCryptoInfo(
DecoderInputBuffer buffer, int adaptiveReconfigurationBytes) {
MediaCodec.CryptoInfo cryptoInfo = buffer.cryptoInfo.getFrameworkCryptoInfo();
if (adaptiveReconfigurationBytes == 0) {
return cryptoInfo;
}
// There must be at least one sub-sample, although numBytesOfClearData is permitted to be
// null if it contains no clear data. Instantiate it if needed, and add the reconfiguration
// bytes to the clear byte count of the first sub-sample.
if (cryptoInfo.numBytesOfClearData == null) {
cryptoInfo.numBytesOfClearData = new int[1];
}
cryptoInfo.numBytesOfClearData[0] += adaptiveReconfigurationBytes;
return cryptoInfo;
}
private static boolean isMediaCodecException(IllegalStateException error) { private static boolean isMediaCodecException(IllegalStateException error) {
if (Util.SDK_INT >= 21 && isMediaCodecExceptionV21(error)) { if (Util.SDK_INT >= 21 && isMediaCodecExceptionV21(error)) {
return true; return true;
......
...@@ -27,6 +27,7 @@ import androidx.annotation.Nullable; ...@@ -27,6 +27,7 @@ import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi; import androidx.annotation.RequiresApi;
import androidx.annotation.VisibleForTesting; import androidx.annotation.VisibleForTesting;
import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.decoder.CryptoInfo;
import com.google.android.exoplayer2.util.IntArrayQueue; import com.google.android.exoplayer2.util.IntArrayQueue;
import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.util.Util;
import java.util.ArrayDeque; import java.util.ArrayDeque;
...@@ -169,10 +170,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -169,10 +170,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@Override @Override
public void queueSecureInputBuffer( public void queueSecureInputBuffer(
int index, int offset, MediaCodec.CryptoInfo info, long presentationTimeUs, int flags) { int index, int offset, CryptoInfo info, long presentationTimeUs, int flags) {
// This method does not need to be synchronized because it is not interacting with // This method does not need to be synchronized because it is not interacting with
// MediaCodec.Callback and dequeueing buffers operations. // MediaCodec.Callback and dequeueing buffers operations.
codec.queueSecureInputBuffer(index, offset, info, presentationTimeUs, flags); codec.queueSecureInputBuffer(
index, offset, info.getFrameworkCryptoInfo(), presentationTimeUs, flags);
} }
@Override @Override
......
...@@ -18,6 +18,7 @@ package com.google.android.exoplayer2.mediacodec; ...@@ -18,6 +18,7 @@ package com.google.android.exoplayer2.mediacodec;
import android.media.MediaCodec; import android.media.MediaCodec;
import android.media.MediaFormat; import android.media.MediaFormat;
import com.google.android.exoplayer2.decoder.CryptoInfo;
/** /**
* A {@link MediaCodecAdapter} that operates the underlying {@link MediaCodec} in synchronous mode. * A {@link MediaCodecAdapter} that operates the underlying {@link MediaCodec} in synchronous mode.
...@@ -58,8 +59,9 @@ import android.media.MediaFormat; ...@@ -58,8 +59,9 @@ import android.media.MediaFormat;
@Override @Override
public void queueSecureInputBuffer( public void queueSecureInputBuffer(
int index, int offset, MediaCodec.CryptoInfo info, long presentationTimeUs, int flags) { int index, int offset, CryptoInfo info, long presentationTimeUs, int flags) {
codec.queueSecureInputBuffer(index, offset, info, presentationTimeUs, flags); codec.queueSecureInputBuffer(
index, offset, info.getFrameworkCryptoInfo(), presentationTimeUs, flags);
} }
@Override @Override
......
...@@ -17,6 +17,7 @@ ...@@ -17,6 +17,7 @@
package com.google.android.exoplayer2.mediacodec; package com.google.android.exoplayer2.mediacodec;
import android.media.MediaCodec; import android.media.MediaCodec;
import com.google.android.exoplayer2.decoder.CryptoInfo;
/** /**
* A {@link MediaCodecInputBufferEnqueuer} that forwards queueing methods directly to {@link * A {@link MediaCodecInputBufferEnqueuer} that forwards queueing methods directly to {@link
...@@ -45,8 +46,9 @@ class SynchronousMediaCodecBufferEnqueuer implements MediaCodecInputBufferEnqueu ...@@ -45,8 +46,9 @@ class SynchronousMediaCodecBufferEnqueuer implements MediaCodecInputBufferEnqueu
@Override @Override
public void queueSecureInputBuffer( public void queueSecureInputBuffer(
int index, int offset, MediaCodec.CryptoInfo info, long presentationTimeUs, int flags) { int index, int offset, CryptoInfo info, long presentationTimeUs, int flags) {
codec.queueSecureInputBuffer(index, offset, info, presentationTimeUs, flags); codec.queueSecureInputBuffer(
index, offset, info.getFrameworkCryptoInfo(), presentationTimeUs, flags);
} }
@Override @Override
......
/*
* Copyright (C) 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.mediacodec;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;
import static org.mockito.Mockito.doAnswer;
import android.media.MediaCodec;
import android.os.HandlerThread;
import android.os.Looper;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.decoder.CryptoInfo;
import com.google.android.exoplayer2.util.ConditionVariable;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicLong;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnit;
import org.mockito.junit.MockitoRule;
import org.robolectric.Shadows;
import org.robolectric.shadows.ShadowLooper;
/** Unit tests for {@link AsynchronousMediaCodecBufferEnqueuer}. */
@RunWith(AndroidJUnit4.class)
public class AsynchronousMediaCodecBufferEnqueuerTest {
@Rule public final MockitoRule mockito = MockitoJUnit.rule();
private AsynchronousMediaCodecBufferEnqueuer enqueuer;
private TestHandlerThread handlerThread;
@Mock private ConditionVariable mockConditionVariable;
@Before
public void setUp() throws IOException {
MediaCodec codec = MediaCodec.createByCodecName("h264");
handlerThread = new TestHandlerThread("TestHandlerThread");
enqueuer =
new AsynchronousMediaCodecBufferEnqueuer(codec, handlerThread, mockConditionVariable);
}
@After
public void tearDown() {
enqueuer.shutdown();
assertThat(TestHandlerThread.INSTANCES_STARTED.get()).isEqualTo(0);
}
@Test
public void queueInputBuffer_withPendingCryptoExceptionSet_throwsCryptoException() {
enqueuer.setPendingRuntimeException(
new MediaCodec.CryptoException(/* errorCode= */ 0, /* detailMessage= */ null));
enqueuer.start();
assertThrows(
MediaCodec.CryptoException.class,
() ->
enqueuer.queueInputBuffer(
/* index= */ 0,
/* offset= */ 0,
/* size= */ 0,
/* presentationTimeUs= */ 0,
/* flags= */ 0));
}
@Test
public void queueInputBuffer_withPendingIllegalStateExceptionSet_throwsIllegalStateException() {
enqueuer.start();
enqueuer.setPendingRuntimeException(new IllegalStateException());
assertThrows(
IllegalStateException.class,
() ->
enqueuer.queueInputBuffer(
/* index= */ 0,
/* offset= */ 0,
/* size= */ 0,
/* presentationTimeUs= */ 0,
/* flags= */ 0));
}
@Test
public void queueInputBuffer_multipleTimes_limitsObjectsAllocation() {
enqueuer.start();
Looper looper = handlerThread.getLooper();
ShadowLooper shadowLooper = Shadows.shadowOf(looper);
for (int cycle = 0; cycle < 100; cycle++) {
// Enqueue 10 messages to looper.
for (int i = 0; i < 10; i++) {
enqueuer.queueInputBuffer(
/* index= */ i,
/* offset= */ 0,
/* size= */ 0,
/* presentationTimeUs= */ i,
/* flags= */ 0);
}
// Execute all messages.
shadowLooper.idle();
}
assertThat(AsynchronousMediaCodecBufferEnqueuer.getInstancePoolSize()).isEqualTo(10);
}
@Test
public void queueSecureInputBuffer_withPendingCryptoException_throwsCryptoException() {
enqueuer.setPendingRuntimeException(
new MediaCodec.CryptoException(/* errorCode= */ 0, /* detailMessage= */ null));
enqueuer.start();
CryptoInfo info = createCryptoInfo();
assertThrows(
MediaCodec.CryptoException.class,
() ->
enqueuer.queueSecureInputBuffer(
/* index= */ 0,
/* offset= */ 0,
/* info= */ info,
/* presentationTimeUs= */ 0,
/* flags= */ 0));
}
@Test
public void queueSecureInputBuffer_codecThrewIllegalStateException_throwsIllegalStateException() {
enqueuer.setPendingRuntimeException(new IllegalStateException());
enqueuer.start();
CryptoInfo info = createCryptoInfo();
assertThrows(
IllegalStateException.class,
() ->
enqueuer.queueSecureInputBuffer(
/* index= */ 0,
/* offset= */ 0,
/* info= */ info,
/* presentationTimeUs= */ 0,
/* flags= */ 0));
}
@Test
public void queueSecureInputBuffer_multipleTimes_limitsObjectsAllocation() {
enqueuer.start();
Looper looper = handlerThread.getLooper();
CryptoInfo info = createCryptoInfo();
ShadowLooper shadowLooper = Shadows.shadowOf(looper);
for (int cycle = 0; cycle < 100; cycle++) {
// Enqueue 10 messages to looper.
for (int i = 0; i < 10; i++) {
enqueuer.queueSecureInputBuffer(
/* index= */ i,
/* offset= */ 0,
/* info= */ info,
/* presentationTimeUs= */ i,
/* flags= */ 0);
}
// Execute all messages.
shadowLooper.idle();
}
assertThat(AsynchronousMediaCodecBufferEnqueuer.getInstancePoolSize()).isEqualTo(10);
}
@Test
public void flush_withoutStart_works() {
enqueuer.flush();
}
@Test
public void flush_onInterruptedException_throwsIllegalStateException()
throws InterruptedException {
doAnswer(
invocation -> {
throw new InterruptedException();
})
.doNothing()
.when(mockConditionVariable)
.block();
enqueuer.start();
assertThrows(IllegalStateException.class, () -> enqueuer.flush());
}
@Test
public void flush_multipleTimes_works() {
enqueuer.start();
enqueuer.flush();
enqueuer.flush();
}
@Test
public void shutdown_withoutStart_works() {
enqueuer.shutdown();
}
@Test
public void shutdown_multipleTimes_works() {
enqueuer.start();
enqueuer.shutdown();
enqueuer.shutdown();
}
@Test
public void shutdown_onInterruptedException_throwsIllegalStateException()
throws InterruptedException {
doAnswer(
invocation -> {
throw new InterruptedException();
})
.doNothing()
.when(mockConditionVariable)
.block();
enqueuer.start();
assertThrows(IllegalStateException.class, () -> enqueuer.shutdown());
}
private static class TestHandlerThread extends HandlerThread {
private static final AtomicLong INSTANCES_STARTED = new AtomicLong(0);
TestHandlerThread(String name) {
super(name);
}
@Override
public synchronized void start() {
super.start();
INSTANCES_STARTED.incrementAndGet();
}
@Override
public boolean quit() {
boolean quit = super.quit();
if (quit) {
INSTANCES_STARTED.decrementAndGet();
}
return quit;
}
}
private static CryptoInfo createCryptoInfo() {
CryptoInfo info = new CryptoInfo();
int numSubSamples = 5;
int[] numBytesOfClearData = new int[] {0, 1, 2, 3};
int[] numBytesOfEncryptedData = new int[] {4, 5, 6, 7};
byte[] key = new byte[] {0, 1, 2, 3};
byte[] iv = new byte[] {4, 5, 6, 7};
@C.CryptoMode int mode = C.CRYPTO_MODE_AES_CBC;
int encryptedBlocks = 16;
int clearBlocks = 8;
info.set(
numSubSamples,
numBytesOfClearData,
numBytesOfEncryptedData,
key,
iv,
mode,
encryptedBlocks,
clearBlocks);
return info;
}
}
...@@ -27,6 +27,7 @@ import android.os.Handler; ...@@ -27,6 +27,7 @@ import android.os.Handler;
import android.os.HandlerThread; import android.os.HandlerThread;
import android.os.Looper; import android.os.Looper;
import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.android.exoplayer2.C;
import java.io.IOException; import java.io.IOException;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
...@@ -49,7 +50,12 @@ public class DedicatedThreadAsyncMediaCodecAdapterTest { ...@@ -49,7 +50,12 @@ public class DedicatedThreadAsyncMediaCodecAdapterTest {
public void setUp() throws IOException { public void setUp() throws IOException {
codec = MediaCodec.createByCodecName("h264"); codec = MediaCodec.createByCodecName("h264");
handlerThread = new TestHandlerThread("TestHandlerThread"); handlerThread = new TestHandlerThread("TestHandlerThread");
adapter = new DedicatedThreadAsyncMediaCodecAdapter(codec, handlerThread); adapter =
new DedicatedThreadAsyncMediaCodecAdapter(
codec,
/* enableAsynchronousQueueing= */ false,
/* trackType= */ C.TRACK_TYPE_VIDEO,
handlerThread);
adapter.setCodecStartRunnable(() -> {}); adapter.setCodecStartRunnable(() -> {});
bufferInfo = new MediaCodec.BufferInfo(); bufferInfo = new MediaCodec.BufferInfo();
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment