Enable MediaCodec asynchronous mode

Enable using MediaCodec in async mode. Expose experimental
API to enable/disable the feature.

PiperOrigin-RevId: 283309798

library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecAsyncCallback.java

+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.mediacodec;
+
+import android.media.MediaCodec;
+import android.media.MediaFormat;
+import androidx.annotation.NonNull;
+import androidx.annotation.Nullable;
+import androidx.annotation.RequiresApi;
+import androidx.annotation.VisibleForTesting;
+import com.google.android.exoplayer2.util.IntArrayQueue;
+import java.util.ArrayDeque;
+
+/** Handles the asynchronous callbacks from {@link android.media.MediaCodec.Callback}. */
+@RequiresApi(21)
+/* package */ final class MediaCodecAsyncCallback extends MediaCodec.Callback {
+  private final IntArrayQueue availableInputBuffers;
+  private final IntArrayQueue availableOutputBuffers;
+  private final ArrayDeque<MediaCodec.BufferInfo> bufferInfos;
+  private final ArrayDeque<MediaFormat> formats;
+  @Nullable private MediaFormat currentFormat;
+  @Nullable private IllegalStateException mediaCodecException;
+
+  /** Creates a new MediaCodecAsyncCallback. */
+  public MediaCodecAsyncCallback() {
+    availableInputBuffers = new IntArrayQueue();
+    availableOutputBuffers = new IntArrayQueue();
+    bufferInfos = new ArrayDeque<>();
+    formats = new ArrayDeque<>();
+  }
+
+  /**
+   * Returns the next available input buffer index or {@link MediaCodec#INFO_TRY_AGAIN_LATER} if no
+   * such buffer exists.
+   */
+  public int dequeueInputBufferIndex() {
+    return availableInputBuffers.isEmpty()
+        ? MediaCodec.INFO_TRY_AGAIN_LATER
+        : availableInputBuffers.remove();
+  }
+
+  /**
+   * Returns the next available output buffer index. If the next available output is a MediaFormat
+   * change, it will return {@link MediaCodec#INFO_OUTPUT_FORMAT_CHANGED} and you should call {@link
+   * #getOutputFormat()} to get the format. If there is no available output, this method will return
+   * {@link MediaCodec#INFO_TRY_AGAIN_LATER}.
+   */
+  public int dequeueOutputBufferIndex(MediaCodec.BufferInfo bufferInfo) {
+    if (availableOutputBuffers.isEmpty()) {
+      return MediaCodec.INFO_TRY_AGAIN_LATER;
+    } else {
+      int bufferIndex = availableOutputBuffers.remove();
+      if (bufferIndex >= 0) {
+        MediaCodec.BufferInfo nextBufferInfo = bufferInfos.remove();
+        bufferInfo.set(
+            nextBufferInfo.offset,
+            nextBufferInfo.size,
+            nextBufferInfo.presentationTimeUs,
+            nextBufferInfo.flags);
+      } else if (bufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
+        currentFormat = formats.remove();
+      }
+      return bufferIndex;
+    }
+  }
+
+  /**
+   * Returns the {@link MediaFormat} signalled by the underlying {@link MediaCodec}.
+   *
+   * <p>Call this <b>after</b> {@link #dequeueOutputBufferIndex} returned {@link
+   * MediaCodec#INFO_OUTPUT_FORMAT_CHANGED}.
+   *
+   * @throws {@link IllegalStateException} if you call this method before before {
+   * @link #dequeueOutputBufferIndex} returned {@link MediaCodec#INFO_OUTPUT_FORMAT_CHANGED}.
+   */
+  public MediaFormat getOutputFormat() throws IllegalStateException {
+    if (currentFormat == null) {
+      throw new IllegalStateException();
+    }
+
+    return currentFormat;
+  }
+
+  /**
+   * Checks and throws an {@link IllegalStateException} if an error was previously set on this
+   * instance via {@link #onError}.
+   */
+  public void maybeThrowMediaCodecException() throws IllegalStateException {
+    IllegalStateException exception = mediaCodecException;
+    mediaCodecException = null;
+
+    if (exception != null) {
+      throw exception;
+    }
+  }
+
+  /**
+   * Flushes the MediaCodecAsyncCallback. This method removes all available input and output buffers
+   * and any error that was previously set.
+   */
+  public void flush() {
+    availableInputBuffers.clear();
+    availableOutputBuffers.clear();
+    bufferInfos.clear();
+    formats.clear();
+    mediaCodecException = null;
+  }
+
+  @Override
+  public void onInputBufferAvailable(@NonNull MediaCodec mediaCodec, int i) {
+    availableInputBuffers.add(i);
+  }
+
+  @Override
+  public void onOutputBufferAvailable(
+      @NonNull MediaCodec mediaCodec, int i, @NonNull MediaCodec.BufferInfo bufferInfo) {
+    availableOutputBuffers.add(i);
+    bufferInfos.add(bufferInfo);
+  }
+
+  @Override
+  public void onError(@NonNull MediaCodec mediaCodec, @NonNull MediaCodec.CodecException e) {
+    onMediaCodecError(e);
+  }
+
+  @Override
+  public void onOutputFormatChanged(
+      @NonNull MediaCodec mediaCodec, @NonNull MediaFormat mediaFormat) {
+    availableOutputBuffers.add(MediaCodec.INFO_OUTPUT_FORMAT_CHANGED);
+    formats.add(mediaFormat);
+  }
+
+  @VisibleForTesting()
+  void onMediaCodecError(IllegalStateException e) {
+    mediaCodecException = e;
+  }
+}

library/core/src/main/java/com/google/android/exoplayer2/util/IntArrayQueue.java

+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.util;
+
+import java.util.NoSuchElementException;
+
+/**
+ * Array-based unbounded queue for int primitives with amortized O(1) add and remove.
+ *
+ * <p>Use this class instead of a {@link java.util.Deque} to avoid boxing int primitives to {@link
+ * Integer} instances.
+ */
+public final class IntArrayQueue {
+
+  /** Default capacity needs to be a power of 2. */
+  private static int DEFAULT_INITIAL_CAPACITY = 16;
+
+  private int headIndex;
+  private int tailIndex;
+  private int size;
+  private int[] data;
+  private int wrapAroundMask;
+
+  public IntArrayQueue() {
+    headIndex = 0;
+    tailIndex = -1;
+    size = 0;
+    data = new int[DEFAULT_INITIAL_CAPACITY];
+    wrapAroundMask = data.length - 1;
+  }
+
+  /** Add a new item to the queue. */
+  public void add(int value) {
+    if (size == data.length) {
+      doubleArraySize();
+    }
+
+    tailIndex = (tailIndex + 1) & wrapAroundMask;
+    data[tailIndex] = value;
+    size++;
+  }
+
+  /**
+   * Remove an item from the queue.
+   *
+   * @throws {@link NoSuchElementException} if the queue is empty.
+   */
+  public int remove() {
+    if (size == 0) {
+      throw new NoSuchElementException();
+    }
+
+    int value = data[headIndex];
+    headIndex = (headIndex + 1) & wrapAroundMask;
+    size--;
+
+    return value;
+  }
+
+  /** Returns the number of items in the queue. */
+  public int size() {
+    return size;
+  }
+
+  /** Returns whether the queue is empty. */
+  public boolean isEmpty() {
+    return size == 0;
+  }
+
+  /** Clears the queue. */
+  public void clear() {
+    headIndex = 0;
+    tailIndex = -1;
+    size = 0;
+  }
+
+  /** Returns the length of the backing array. */
+  public int capacity() {
+    return data.length;
+  }
+
+  private void doubleArraySize() {
+    int newCapacity = data.length << 1;
+    if (newCapacity < 0) {
+      throw new IllegalStateException();
+    }
+
+    int[] newData = new int[newCapacity];
+    int itemsToRight = data.length - headIndex;
+    int itemsToLeft = headIndex;
+    System.arraycopy(data, headIndex, newData, 0, itemsToRight);
+    System.arraycopy(data, 0, newData, itemsToRight, itemsToLeft);
+
+    headIndex = 0;
+    tailIndex = size - 1;
+    data = newData;
+    wrapAroundMask = data.length - 1;
+  }
+}

library/core/src/test/java/com/google/android/exoplayer2/mediacodec/MediaCodecAsyncCallbackTest.java

+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.mediacodec;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.fail;
+
+import android.media.MediaCodec;
+import android.media.MediaFormat;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import java.io.IOException;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/** Unit tests for {@link MediaCodecAsyncCallback}. */
+@RunWith(AndroidJUnit4.class)
+public class MediaCodecAsyncCallbackTest {
+
+  private MediaCodecAsyncCallback mediaCodecAsyncCallback;
+  private MediaCodec codec;
+
+  @Before
+  public void setUp() throws IOException {
+    mediaCodecAsyncCallback = new MediaCodecAsyncCallback();
+    codec = MediaCodec.createByCodecName("h264");
+  }
+
+  @Test
+  public void dequeInputBufferIndex_afterCreation_returnsTryAgain() {
+    assertThat(mediaCodecAsyncCallback.dequeueInputBufferIndex())
+        .isEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER);
+  }
+
+  @Test
+  public void dequeInputBufferIndex_returnsEnqueuedBuffers() {
+    // Send two input buffers to the mediaCodecAsyncCallback.
+    mediaCodecAsyncCallback.onInputBufferAvailable(codec, 0);
+    mediaCodecAsyncCallback.onInputBufferAvailable(codec, 1);
+
+    assertThat(mediaCodecAsyncCallback.dequeueInputBufferIndex()).isEqualTo(0);
+    assertThat(mediaCodecAsyncCallback.dequeueInputBufferIndex()).isEqualTo(1);
+    assertThat(mediaCodecAsyncCallback.dequeueInputBufferIndex())
+        .isEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER);
+  }
+
+  @Test
+  public void dequeInputBufferIndex_afterFlush_returnsTryAgain() {
+    // Send two input buffers to the mediaCodecAsyncCallback and then flush().
+    mediaCodecAsyncCallback.onInputBufferAvailable(codec, 0);
+    mediaCodecAsyncCallback.onInputBufferAvailable(codec, 1);
+    mediaCodecAsyncCallback.flush();
+
+    assertThat(mediaCodecAsyncCallback.dequeueInputBufferIndex())
+        .isEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER);
+  }
+
+  @Test
+  public void dequeInputBufferIndex_afterFlushAndNewInputBuffer_returnsEnqueuedBuffer() {
+    // Send two input buffers to the mediaCodecAsyncCallback, then flush(), then send
+    // another input buffer.
+    mediaCodecAsyncCallback.onInputBufferAvailable(codec, 0);
+    mediaCodecAsyncCallback.onInputBufferAvailable(codec, 1);
+    mediaCodecAsyncCallback.flush();
+    mediaCodecAsyncCallback.onInputBufferAvailable(codec, 2);
+
+    assertThat(mediaCodecAsyncCallback.dequeueInputBufferIndex()).isEqualTo(2);
+  }
+
+  @Test
+  public void dequeOutputBufferIndex_afterCreation_returnsTryAgain() {
+    MediaCodec.BufferInfo outBufferInfo = new MediaCodec.BufferInfo();
+    assertThat(mediaCodecAsyncCallback.dequeueOutputBufferIndex(outBufferInfo))
+        .isEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER);
+  }
+
+  @Test
+  public void dequeOutputBufferIndex_returnsEnqueuedBuffers() {
+    // Send two output buffers to the mediaCodecAsyncCallback.
+    MediaCodec.BufferInfo bufferInfo1 = new MediaCodec.BufferInfo();
+    mediaCodecAsyncCallback.onOutputBufferAvailable(codec, 0, bufferInfo1);
+
+    MediaCodec.BufferInfo bufferInfo2 = new MediaCodec.BufferInfo();
+    bufferInfo2.set(1, 1, 1, 1);
+    mediaCodecAsyncCallback.onOutputBufferAvailable(codec, 1, bufferInfo2);
+
+    MediaCodec.BufferInfo outBufferInfo = new MediaCodec.BufferInfo();
+
+    assertThat(mediaCodecAsyncCallback.dequeueOutputBufferIndex(outBufferInfo)).isEqualTo(0);
+    assertThat(areEqual(outBufferInfo, bufferInfo1)).isTrue();
+    assertThat(mediaCodecAsyncCallback.dequeueOutputBufferIndex(outBufferInfo)).isEqualTo(1);
+    assertThat(areEqual(outBufferInfo, bufferInfo2)).isTrue();
+    assertThat(mediaCodecAsyncCallback.dequeueOutputBufferIndex(outBufferInfo))
+        .isEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER);
+  }
+
+  @Test
+  public void dequeOutputBufferIndex_afterFlush_returnsTryAgain() {
+    // Send two output buffers to the mediaCodecAsyncCallback and then flush().
+    MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
+    mediaCodecAsyncCallback.onOutputBufferAvailable(codec, 0, bufferInfo);
+    mediaCodecAsyncCallback.onOutputBufferAvailable(codec, 1, bufferInfo);
+    mediaCodecAsyncCallback.flush();
+    MediaCodec.BufferInfo outBufferInfo = new MediaCodec.BufferInfo();
+
+    assertThat(mediaCodecAsyncCallback.dequeueOutputBufferIndex(outBufferInfo))
+        .isEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER);
+  }
+
+  @Test
+  public void dequeOutputBufferIndex_afterFlushAndNewOutputBuffers_returnsEnqueueBuffer() {
+    // Send two output buffers to the mediaCodecAsyncCallback, then flush(), then send
+    // another output buffer.
+    MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
+    mediaCodecAsyncCallback.onOutputBufferAvailable(codec, 0, bufferInfo);
+    mediaCodecAsyncCallback.onOutputBufferAvailable(codec, 1, bufferInfo);
+    mediaCodecAsyncCallback.flush();
+    mediaCodecAsyncCallback.onOutputBufferAvailable(codec, 2, bufferInfo);
+    MediaCodec.BufferInfo outBufferInfo = new MediaCodec.BufferInfo();
+
+    assertThat(mediaCodecAsyncCallback.dequeueOutputBufferIndex(outBufferInfo)).isEqualTo(2);
+  }
+
+  @Test
+  public void getOutputFormat_onNewInstance_raisesException() {
+    try {
+      mediaCodecAsyncCallback.getOutputFormat();
+      fail();
+    } catch (IllegalStateException expected) {
+    }
+  }
+
+  @Test
+  public void getOutputFormat_afterOnOutputFormatCalled_returnsFormat() {
+    MediaFormat format = new MediaFormat();
+    mediaCodecAsyncCallback.onOutputFormatChanged(codec, format);
+    MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
+
+    assertThat(mediaCodecAsyncCallback.dequeueOutputBufferIndex(bufferInfo))
+        .isEqualTo(MediaCodec.INFO_OUTPUT_FORMAT_CHANGED);
+    assertThat(mediaCodecAsyncCallback.getOutputFormat()).isEqualTo(format);
+  }
+
+  @Test
+  public void getOutputFormat_afterFlush_raisesCurrentFormat() {
+    MediaFormat format = new MediaFormat();
+    mediaCodecAsyncCallback.onOutputFormatChanged(codec, format);
+    MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
+    mediaCodecAsyncCallback.dequeueOutputBufferIndex(bufferInfo);
+    mediaCodecAsyncCallback.flush();
+
+    assertThat(mediaCodecAsyncCallback.getOutputFormat()).isEqualTo(format);
+  }
+
+  @Test
+  public void maybeThrowExoPlaybackException_withoutErrorFromCodec_doesNotThrow() {
+    mediaCodecAsyncCallback.maybeThrowMediaCodecException();
+  }
+
+  @Test
+  public void maybeThrowExoPlaybackException_withErrorFromCodec_Throws() {
+    IllegalStateException exception = new IllegalStateException();
+    mediaCodecAsyncCallback.onMediaCodecError(exception);
+
+    try {
+      mediaCodecAsyncCallback.maybeThrowMediaCodecException();
+      fail();
+    } catch (IllegalStateException expected) {
+    }
+  }
+
+  @Test
+  public void maybeThrowExoPlaybackException_doesNotThrowTwice() {
+    IllegalStateException exception = new IllegalStateException();
+    mediaCodecAsyncCallback.onMediaCodecError(exception);
+
+    try {
+      mediaCodecAsyncCallback.maybeThrowMediaCodecException();
+      fail();
+    } catch (IllegalStateException expected) {
+    }
+
+    mediaCodecAsyncCallback.maybeThrowMediaCodecException();
+  }
+
+  @Test
+  public void maybeThrowExoPlaybackException_afterFlush_doesNotThrow() {
+    IllegalStateException exception = new IllegalStateException();
+    mediaCodecAsyncCallback.onMediaCodecError(exception);
+    mediaCodecAsyncCallback.flush();
+
+    mediaCodecAsyncCallback.maybeThrowMediaCodecException();
+  }
+
+  /**
+   * Compares if two {@link android.media.MediaCodec.BufferInfo} are equal by inspecting {@link
+   * android.media.MediaCodec.BufferInfo#flags}, {@link android.media.MediaCodec.BufferInfo#size},
+   * {@link android.media.MediaCodec.BufferInfo#presentationTimeUs} and {@link
+   * android.media.MediaCodec.BufferInfo#offset}.
+   */
+  private static boolean areEqual(MediaCodec.BufferInfo lhs, MediaCodec.BufferInfo rhs) {
+    return lhs.flags == rhs.flags
+        && lhs.offset == rhs.offset
+        && lhs.presentationTimeUs == rhs.presentationTimeUs
+        && lhs.size == rhs.size;
+  }
+}

library/core/src/test/java/com/google/android/exoplayer2/util/IntArrayQueueTest.java

+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.util;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.fail;
+
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import java.util.NoSuchElementException;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/** Tests for {@link IntArrayQueue}. */
+@RunWith(AndroidJUnit4.class)
+public class IntArrayQueueTest {
+
+  @Test
+  public void add_willDoubleCapacity() {
+    IntArrayQueue queue = new IntArrayQueue();
+    int capacity = queue.capacity();
+
+    for (int i = 0; i <= capacity; i++) {
+      queue.add(i);
+    }
+
+    assertThat(queue.capacity()).isEqualTo(2 * capacity);
+    assertThat(queue.size()).isEqualTo(capacity + 1);
+  }
+
+  @Test
+  public void isEmpty_returnsTrueAfterConstruction() {
+    IntArrayQueue queue = new IntArrayQueue();
+
+    assertThat(queue.isEmpty()).isTrue();
+  }
+
+  @Test
+  public void isEmpty_returnsFalseAfterAddition() {
+    IntArrayQueue queue = new IntArrayQueue();
+    queue.add(0);
+
+    assertThat(queue.isEmpty()).isFalse();
+  }
+
+  @Test
+  public void isEmpty_returnsFalseAfterRemoval() {
+    IntArrayQueue queue = new IntArrayQueue();
+    queue.add(0);
+    queue.remove();
+
+    assertThat(queue.isEmpty()).isTrue();
+  }
+
+  @Test
+  public void remove_onEmptyQueue_throwsException() {
+    IntArrayQueue queue = new IntArrayQueue();
+
+    try {
+      queue.remove();
+      fail();
+    } catch (NoSuchElementException expected) {
+      // expected
+    }
+  }
+
+  @Test
+  public void remove_returnsCorrectItem() {
+    IntArrayQueue queue = new IntArrayQueue();
+    int value = 20;
+    queue.add(value);
+
+    assertThat(queue.remove()).isEqualTo(value);
+  }
+
+  @Test
+  public void remove_untilIsEmpty() {
+    IntArrayQueue queue = new IntArrayQueue();
+    for (int i = 0; i < 1024; i++) {
+      queue.add(i);
+    }
+
+    int expectedRemoved = 0;
+    while (!queue.isEmpty()) {
+      if (expectedRemoved == 15) {
+        System.out.println("foo");
+      }
+      int removed = queue.remove();
+      assertThat(removed).isEqualTo(expectedRemoved++);
+    }
+  }
+
+  @Test
+  public void remove_withResize_returnsCorrectItem() {
+    IntArrayQueue queue = new IntArrayQueue();
+    int nextToAdd = 0;
+
+    while (queue.size() < queue.capacity()) {
+      queue.add(nextToAdd++);
+    }
+
+    queue.remove();
+    queue.remove();
+
+    // This will force the queue to wrap-around and then resize
+    int howManyToResize = queue.capacity() - queue.size() + 1;
+    for (int i = 0; i < howManyToResize; i++) {
+      queue.add(nextToAdd++);
+    }
+
+    assertThat(queue.remove()).isEqualTo(2);
+  }
+
+  @Test
+  public void clear_resetsQueue() {
+    IntArrayQueue queue = new IntArrayQueue();
+
+    // Add items until array re-sizes twice (capacity grows by 4)
+    for (int i = 0; i < 1024; i++) {
+      queue.add(i);
+    }
+
+    queue.clear();
+
+    assertThat(queue.size()).isEqualTo(0);
+    assertThat(queue.isEmpty()).isTrue();
+  }
+}