Split ExoPlayerAssetLoaderRenderer

Split ExoPlayerAssetLoaderRenderer into audio and video renderers.

PiperOrigin-RevId: 500102256

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoAssetLoaderAudioRenderer.java

+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.transformer;
+
+import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
+
+import android.media.MediaCodec;
+import androidx.annotation.Nullable;
+import com.google.android.exoplayer2.C;
+import com.google.android.exoplayer2.Format;
+import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
+import java.nio.ByteBuffer;
+import org.checkerframework.checker.nullness.qual.RequiresNonNull;
+
+/* package */ final class ExoAssetLoaderAudioRenderer extends ExoAssetLoaderBaseRenderer {
+
+  private static final String TAG = "ExoAssetLoaderAudioRenderer";
+
+  private final Codec.DecoderFactory decoderFactory;
+
+  @Nullable private ByteBuffer pendingDecoderOutputBuffer;
+
+  public ExoAssetLoaderAudioRenderer(
+      Codec.DecoderFactory decoderFactory,
+      TransformerMediaClock mediaClock,
+      AssetLoader.Listener assetLoaderListener) {
+    super(C.TRACK_TYPE_AUDIO, mediaClock, assetLoaderListener);
+    this.decoderFactory = decoderFactory;
+  }
+
+  @Override
+  public String getName() {
+    return TAG;
+  }
+
+  @Override
+  protected void initDecoder(Format inputFormat) throws TransformationException {
+    decoder = decoderFactory.createForAudioDecoding(inputFormat);
+  }
+
+  /**
+   * Attempts to get decoded audio data and pass it to the sample consumer.
+   *
+   * @return Whether it may be possible to read more data immediately by calling this method again.
+   * @throws TransformationException If an error occurs in the decoder.
+   */
+  @Override
+  @RequiresNonNull("sampleConsumer")
+  protected boolean feedConsumerFromDecoder() throws TransformationException {
+    @Nullable DecoderInputBuffer sampleConsumerInputBuffer = sampleConsumer.dequeueInputBuffer();
+    if (sampleConsumerInputBuffer == null) {
+      return false;
+    }
+
+    Codec decoder = checkNotNull(this.decoder);
+    if (pendingDecoderOutputBuffer != null) {
+      if (pendingDecoderOutputBuffer.hasRemaining()) {
+        return false;
+      } else {
+        decoder.releaseOutputBuffer(/* render= */ false);
+        pendingDecoderOutputBuffer = null;
+      }
+    }
+
+    if (decoder.isEnded()) {
+      sampleConsumerInputBuffer.addFlag(C.BUFFER_FLAG_END_OF_STREAM);
+      sampleConsumer.queueInputBuffer();
+      isEnded = true;
+      return false;
+    }
+
+    pendingDecoderOutputBuffer = decoder.getOutputBuffer();
+    if (pendingDecoderOutputBuffer == null) {
+      return false;
+    }
+
+    sampleConsumerInputBuffer.data = pendingDecoderOutputBuffer;
+    MediaCodec.BufferInfo bufferInfo = checkNotNull(decoder.getOutputBufferInfo());
+    sampleConsumerInputBuffer.timeUs = bufferInfo.presentationTimeUs;
+    sampleConsumerInputBuffer.setFlags(bufferInfo.flags);
+    sampleConsumer.queueInputBuffer();
+    return true;
+  }
+}

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoAssetLoaderVideoRenderer.java

+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.transformer;
+
+import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
+
+import android.media.MediaCodec;
+import androidx.annotation.Nullable;
+import com.google.android.exoplayer2.C;
+import com.google.android.exoplayer2.Format;
+import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
+import com.google.android.exoplayer2.video.ColorInfo;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+import org.checkerframework.checker.nullness.qual.RequiresNonNull;
+
+/* package */ final class ExoAssetLoaderVideoRenderer extends ExoAssetLoaderBaseRenderer {
+
+  private static final String TAG = "ExoAssetLoaderVideoRenderer";
+
+  private final boolean flattenForSlowMotion;
+  private final Codec.DecoderFactory decoderFactory;
+  private final List<Long> decodeOnlyPresentationTimestamps;
+
+  private @MonotonicNonNull SefSlowMotionFlattener sefVideoSlowMotionFlattener;
+  private int maxDecoderPendingFrameCount;
+
+  public ExoAssetLoaderVideoRenderer(
+      boolean flattenForSlowMotion,
+      Codec.DecoderFactory decoderFactory,
+      TransformerMediaClock mediaClock,
+      AssetLoader.Listener assetLoaderListener) {
+    super(C.TRACK_TYPE_VIDEO, mediaClock, assetLoaderListener);
+    this.flattenForSlowMotion = flattenForSlowMotion;
+    this.decoderFactory = decoderFactory;
+    decodeOnlyPresentationTimestamps = new ArrayList<>();
+  }
+
+  @Override
+  public String getName() {
+    return TAG;
+  }
+
+  @Override
+  protected void onInputFormatRead(Format inputFormat) {
+    if (flattenForSlowMotion) {
+      sefVideoSlowMotionFlattener = new SefSlowMotionFlattener(inputFormat);
+    }
+  }
+
+  @Override
+  @RequiresNonNull("sampleConsumer")
+  protected void initDecoder(Format inputFormat) throws TransformationException {
+    boolean isDecoderToneMappingRequired =
+        ColorInfo.isTransferHdr(inputFormat.colorInfo)
+            && !ColorInfo.isTransferHdr(sampleConsumer.getExpectedColorInfo());
+    decoder =
+        decoderFactory.createForVideoDecoding(
+            inputFormat,
+            checkNotNull(sampleConsumer.getInputSurface()),
+            isDecoderToneMappingRequired);
+    maxDecoderPendingFrameCount = decoder.getMaxPendingFrameCount();
+  }
+
+  @Override
+  protected boolean shouldDropInputBuffer(DecoderInputBuffer inputBuffer) {
+    ByteBuffer inputBytes = checkNotNull(inputBuffer.data);
+
+    if (sefVideoSlowMotionFlattener == null || inputBuffer.isEndOfStream()) {
+      return false;
+    }
+
+    long presentationTimeUs = inputBuffer.timeUs - streamOffsetUs;
+    boolean shouldDropInputBuffer =
+        sefVideoSlowMotionFlattener.dropOrTransformSample(inputBytes, presentationTimeUs);
+    if (shouldDropInputBuffer) {
+      inputBytes.clear();
+    } else {
+      inputBuffer.timeUs =
+          streamOffsetUs + sefVideoSlowMotionFlattener.getSamplePresentationTimeUs();
+    }
+    return shouldDropInputBuffer;
+  }
+
+  @Override
+  protected void onDecoderInputReady(DecoderInputBuffer inputBuffer) {
+    if (inputBuffer.isDecodeOnly()) {
+      decodeOnlyPresentationTimestamps.add(inputBuffer.timeUs);
+    }
+  }
+
+  @Override
+  @RequiresNonNull("sampleConsumer")
+  protected boolean feedConsumerFromDecoder() throws TransformationException {
+    Codec decoder = checkNotNull(this.decoder);
+    if (decoder.isEnded()) {
+      sampleConsumer.signalEndOfVideoInput();
+      isEnded = true;
+      return false;
+    }
+
+    @Nullable MediaCodec.BufferInfo decoderOutputBufferInfo = decoder.getOutputBufferInfo();
+    if (decoderOutputBufferInfo == null) {
+      return false;
+    }
+
+    if (isDecodeOnlyBuffer(decoderOutputBufferInfo.presentationTimeUs)) {
+      decoder.releaseOutputBuffer(/* render= */ false);
+      return true;
+    }
+
+    if (maxDecoderPendingFrameCount != C.UNLIMITED_PENDING_FRAME_COUNT
+        && sampleConsumer.getPendingVideoFrameCount() == maxDecoderPendingFrameCount) {
+      return false;
+    }
+
+    sampleConsumer.registerVideoFrame();
+    decoder.releaseOutputBuffer(/* render= */ true);
+    return true;
+  }
+
+  private boolean isDecodeOnlyBuffer(long presentationTimeUs) {
+    // We avoid using decodeOnlyPresentationTimestamps.remove(presentationTimeUs) because it would
+    // box presentationTimeUs, creating a Long object that would need to be garbage collected.
+    int size = decodeOnlyPresentationTimestamps.size();
+    for (int i = 0; i < size; i++) {
+      if (decodeOnlyPresentationTimestamps.get(i) == presentationTimeUs) {
+        decodeOnlyPresentationTimestamps.remove(i);
+        return true;
+      }
+    }
+    return false;
+  }
+}

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoPlayerAssetLoader.java

@@ -291,22 +291,13 @@ public final class ExoPlayerAssetLoader implements AssetLoader {
       int index = 0;
       int index = 0;
       if (!removeAudio) {
       if (!removeAudio) {
         renderers[index] =
         renderers[index] =
-            new ExoPlayerAssetLoaderRenderer(
-                C.TRACK_TYPE_AUDIO,
-                /* flattenForSlowMotion= */ false,
-                decoderFactory,
-                mediaClock,
-                assetLoaderListener);
+            new ExoAssetLoaderAudioRenderer(decoderFactory, mediaClock, assetLoaderListener);
         index++;
         index++;
       }
       }
       if (!removeVideo) {
       if (!removeVideo) {
         renderers[index] =
         renderers[index] =
-            new ExoPlayerAssetLoaderRenderer(
-                C.TRACK_TYPE_VIDEO,
-                flattenForSlowMotion,
-                decoderFactory,
-                mediaClock,
-                assetLoaderListener);
+            new ExoAssetLoaderVideoRenderer(
+                flattenForSlowMotion, decoderFactory, mediaClock, assetLoaderListener);
         index++;
         index++;
       }
       }
       return renderers;
       return renderers;