Prototype video transcoding

The prototype is built upon Transformer and took many references from
TransformerAudioRenderer.

Please take a look and we can discuss more details.

PiperOrigin-RevId: 390192487

demos/main/src/main/assets/media.exolist.json

@@ -4,7 +4,7 @@
     "samples": [
       {
         "name": "HD (MP4, H264)",
-        "uri": "https://storage.googleapis.com/wvmedia/clear/h264/tears/tears.mpd"
+        "uri": "rtsp://localhost:15554"
       },
       {
         "name": "UHD (MP4, H264)",

library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecAdapter.java

@@ -45,7 +45,10 @@ public interface MediaCodecAdapter {
     public final MediaFormat mediaFormat;
     /** The {@link Format} for which the codec is being configured. */
     public final Format format;
-    /** For video playbacks, the output where the object will render the decoded frames. */
+    /**
+     * For video decoding, the output where the object will render the decoded frames; for video
+     * encoding, this is the input surface from which the decoded frames are retrieved.
+     */
     @Nullable public final Surface surface;
     /** For DRM protected playbacks, a {@link MediaCrypto} to use for decryption. */
     @Nullable public final MediaCrypto crypto;

library/core/src/main/java/com/google/android/exoplayer2/mediacodec/SynchronousMediaCodecAdapter.java

@@ -21,9 +21,11 @@ import static com.google.android.exoplayer2.util.Util.castNonNull;
 
 import android.media.MediaCodec;
 import android.media.MediaFormat;
+import android.os.Build;
 import android.os.Bundle;
 import android.os.Handler;
 import android.view.Surface;
+import androidx.annotation.DoNotInline;
 import androidx.annotation.Nullable;
 import androidx.annotation.RequiresApi;
 import com.google.android.exoplayer2.C;
@@ -42,17 +44,29 @@ public class SynchronousMediaCodecAdapter implements MediaCodecAdapter {
   public static class Factory implements MediaCodecAdapter.Factory {
 
     @Override
+    @RequiresApi(16)
     public MediaCodecAdapter createAdapter(Configuration configuration) throws IOException {
       @Nullable MediaCodec codec = null;
+      boolean isEncoder = configuration.flags == MediaCodec.CONFIGURE_FLAG_ENCODE;
+      @Nullable Surface decoderOutputSurface = isEncoder ? null : configuration.surface;
       try {
         codec = createCodec(configuration);
         TraceUtil.beginSection("configureCodec");
         codec.configure(
             configuration.mediaFormat,
-            configuration.surface,
+            decoderOutputSurface,
             configuration.crypto,
             configuration.flags);
         TraceUtil.endSection();
+        if (isEncoder && configuration.surface != null) {
+          if (Build.VERSION.SDK_INT >= 23) {
+            Api23.setCodecInputSurface(codec, configuration.surface);
+          } else {
+            throw new IllegalStateException(
+                "Encoding from a surface is only supported on API 23 and up");
+          }
+        }
+
         TraceUtil.beginSection("startCodec");
         codec.start();
         TraceUtil.endSection();
@@ -198,4 +212,12 @@ public class SynchronousMediaCodecAdapter implements MediaCodecAdapter {
   public void setVideoScalingMode(@C.VideoScalingMode int scalingMode) {
     codec.setVideoScalingMode(scalingMode);
   }
+
+  @RequiresApi(23)
+  private static final class Api23 {
+    @DoNotInline
+    public static void setCodecInputSurface(MediaCodec codec, Surface surface) {
+      codec.setInputSurface(surface);
+    }
+  }
 }

library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/RtspMediaSource.java

@@ -69,7 +69,7 @@ public final class RtspMediaSource extends BaseMediaSource {
     private long timeoutMs;
     private String userAgent;
     private boolean forceUseRtpTcp;
-    private boolean debugLoggingEnabled;
+    private boolean debugLoggingEnabled = true;
 
     public Factory() {
       timeoutMs = DEFAULT_TIMEOUT_MS;

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MediaCodecAdapterWrapper.java

@@ -21,8 +21,11 @@ import static com.google.android.exoplayer2.util.Assertions.checkState;
 
 import android.media.MediaCodec;
 import android.media.MediaCodec.BufferInfo;
+import android.media.MediaCodecInfo.CodecCapabilities;
 import android.media.MediaFormat;
+import android.view.Surface;
 import androidx.annotation.Nullable;
+import androidx.annotation.RequiresApi;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
@@ -131,6 +134,47 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
   /**
    * Returns a {@link MediaCodecAdapterWrapper} for a configured and started {@link
+   * MediaCodecAdapter} video decoder.
+   *
+   * @param format The {@link Format} (of the input data) used to determine the underlying {@link
+   *     MediaCodec} and its configuration values.
+   * @param surface The {@link Surface} to which the decoder output is rendered.
+   * @return A configured and started decoder wrapper.
+   * @throws IOException If the underlying codec cannot be created.
+   */
+  @RequiresApi(23)
+  public static MediaCodecAdapterWrapper createForVideoDecoding(Format format, Surface surface)
+      throws IOException {
+    @Nullable MediaCodecAdapter adapter = null;
+    try {
+      MediaFormat mediaFormat =
+          MediaFormat.createVideoFormat(
+              checkNotNull(format.sampleMimeType), format.width, format.height);
+      MediaFormatUtil.maybeSetInteger(
+          mediaFormat, MediaFormat.KEY_MAX_INPUT_SIZE, format.maxInputSize);
+      MediaFormatUtil.setCsdBuffers(mediaFormat, format.initializationData);
+      adapter =
+          new Factory(/* decoder= */ true)
+              .createAdapter(
+                  new MediaCodecAdapter.Configuration(
+                      createPlaceholderMediaCodecInfo(),
+                      mediaFormat,
+                      format,
+                      surface,
+                      /* crypto= */ null,
+                      /* flags= */ 0));
+      adapter.setOutputSurface(surface);
+      return new MediaCodecAdapterWrapper(adapter);
+    } catch (Exception e) {
+      if (adapter != null) {
+        adapter.release();
+      }
+      throw e;
+    }
+  }
+
+  /**
+   * Returns a {@link MediaCodecAdapterWrapper} for a configured and started {@link
    * MediaCodecAdapter} audio encoder.
    *
    * @param format The {@link Format} (of the output data) used to determine the underlying {@link
@@ -167,6 +211,49 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     }
   }
 
+  /**
+   * Returns a {@link MediaCodecAdapterWrapper} for a configured and started {@link
+   * MediaCodecAdapter} video encoder.
+   *
+   * @param format The {@link Format} (of the output data) used to determine the underlying {@link
+   *     MediaCodec} and its configuration values.
+   * @param surface The {@link Surface} from which the encoder obtains the frame input.
+   * @return A configured and started encoder wrapper.
+   * @throws IOException If the underlying codec cannot be created.
+   */
+  @RequiresApi(18)
+  public static MediaCodecAdapterWrapper createForVideoEncoding(Format format, Surface surface)
+      throws IOException {
+    @Nullable MediaCodecAdapter adapter = null;
+    try {
+      MediaFormat mediaFormat =
+          MediaFormat.createVideoFormat(
+              checkNotNull(format.sampleMimeType), format.width, format.height);
+      // TODO(claincly): enable configuration.
+      mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, CodecCapabilities.COLOR_FormatSurface);
+      mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 30);
+      mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1);
+      mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, 5_000_000);
+
+      adapter =
+          new Factory(/* decoder= */ false)
+              .createAdapter(
+                  new MediaCodecAdapter.Configuration(
+                      createPlaceholderMediaCodecInfo(),
+                      mediaFormat,
+                      format,
+                      surface,
+                      /* crypto= */ null,
+                      MediaCodec.CONFIGURE_FLAG_ENCODE));
+      return new MediaCodecAdapterWrapper(adapter);
+    } catch (Exception e) {
+      if (adapter != null) {
+        adapter.release();
+      }
+      throw e;
+    }
+  }
+
   private MediaCodecAdapterWrapper(MediaCodecAdapter codec) {
     this.codec = codec;
     outputBufferInfo = new BufferInfo();
@@ -232,13 +319,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   /** Returns the current output {@link ByteBuffer}, if available. */
   @Nullable
   public ByteBuffer getOutputBuffer() {
-    return maybeDequeueOutputBuffer() ? outputBuffer : null;
+    return maybeDequeueAndSetOutputBuffer() ? outputBuffer : null;
   }
 
   /** Returns the {@link BufferInfo} associated with the current output buffer, if available. */
   @Nullable
   public BufferInfo getOutputBufferInfo() {
-    return maybeDequeueOutputBuffer() ? outputBufferInfo : null;
+    return maybeDequeueAndSetOutputBuffer() ? outputBufferInfo : null;
   }
 
   /**
@@ -264,6 +351,34 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     codec.release();
   }
 
+  /** Returns {@code true} if a buffer is successfully obtained, rendered and released. */
+  public boolean maybeDequeueRenderAndReleaseOutputBuffer() {
+    if (!maybeDequeueOutputBuffer()) {
+      return false;
+    }
+
+    codec.releaseOutputBuffer(outputBufferIndex, /* render= */ true);
+    outputBuffer = null;
+    outputBufferIndex = C.INDEX_UNSET;
+    return true;
+  }
+
+  /**
+   * Tries obtaining an output buffer and sets {@link #outputBuffer} to the obtained output buffer.
+   *
+   * @return {@code true} if a buffer is successfully obtained, {@code false} otherwise.
+   */
+  private boolean maybeDequeueAndSetOutputBuffer() {
+    if (!maybeDequeueOutputBuffer()) {
+      return false;
+    }
+
+    outputBuffer = checkNotNull(codec.getOutputBuffer(outputBufferIndex));
+    outputBuffer.position(outputBufferInfo.offset);
+    outputBuffer.limit(outputBufferInfo.offset + outputBufferInfo.size);
+    return true;
+  }
+
   /**
    * Returns true if there is already an output buffer pending. Otherwise attempts to dequeue an
    * output buffer and returns whether there is a new output buffer.
@@ -295,11 +410,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       releaseOutputBuffer();
       return false;
     }
-
-    outputBuffer = checkNotNull(codec.getOutputBuffer(outputBufferIndex));
-    outputBuffer.position(outputBufferInfo.offset);
-    outputBuffer.limit(outputBufferInfo.offset + outputBufferInfo.size);
-
     return true;
   }
 

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerTranscodingVideoRenderer.java

+/*
+ * Copyright 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.android.exoplayer2.transformer;
+
+import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
+
+import android.media.MediaCodec;
+import android.view.Surface;
+import androidx.annotation.Nullable;
+import androidx.annotation.RequiresApi;
+import com.google.android.exoplayer2.C;
+import com.google.android.exoplayer2.ExoPlaybackException;
+import com.google.android.exoplayer2.Format;
+import com.google.android.exoplayer2.FormatHolder;
+import com.google.android.exoplayer2.PlaybackException;
+import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
+import com.google.android.exoplayer2.source.SampleStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+@RequiresApi(23)
+/* package */ final class TransformerTranscodingVideoRenderer extends TransformerBaseRenderer {
+
+  private static final String TAG = "TransformerTranscodingVideoRenderer";
+
+  private final DecoderInputBuffer buffer;
+  /** The format the encoder is configured to output, may differ from the actual output format. */
+  private final Format encoderConfigurationOutputFormat;
+
+  private final Surface surface;
+
+  @Nullable private MediaCodecAdapterWrapper decoder;
+  @Nullable private MediaCodecAdapterWrapper encoder;
+  /** Whether encoder's actual output format is obtained. */
+  private boolean hasEncoderActualOutputFormat;
+
+  private boolean muxerWrapperTrackEnded;
+
+  public TransformerTranscodingVideoRenderer(
+      MuxerWrapper muxerWrapper,
+      TransformerMediaClock mediaClock,
+      Transformation transformation,
+      Format encoderConfigurationOutputFormat) {
+    super(C.TRACK_TYPE_VIDEO, muxerWrapper, mediaClock, transformation);
+
+    buffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT);
+    surface = MediaCodec.createPersistentInputSurface();
+    this.encoderConfigurationOutputFormat = encoderConfigurationOutputFormat;
+  }
+
+  @Override
+  public String getName() {
+    return TAG;
+  }
+
+  @Override
+  public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException {
+    if (!isRendererStarted || isEnded()) {
+      return;
+    }
+
+    if (!ensureDecoderConfigured()) {
+      return;
+    }
+
+    if (ensureEncoderConfigured()) {
+      while (feedMuxerFromEncoder()) {}
+      while (feedEncoderFromDecoder()) {}
+    }
+    while (feedDecoderFromInput()) {}
+  }
+
+  @Override
+  public boolean isEnded() {
+    return muxerWrapperTrackEnded;
+  }
+
+  private boolean ensureDecoderConfigured() throws ExoPlaybackException {
+    if (decoder != null) {
+      return true;
+    }
+
+    FormatHolder formatHolder = getFormatHolder();
+    @SampleStream.ReadDataResult
+    int result =
+        readSource(formatHolder, buffer, /* readFlags= */ SampleStream.FLAG_REQUIRE_FORMAT);
+    if (result != C.RESULT_FORMAT_READ) {
+      return false;
+    }
+
+    Format inputFormat = checkNotNull(formatHolder.format);
+    try {
+      decoder = MediaCodecAdapterWrapper.createForVideoDecoding(inputFormat, surface);
+    } catch (IOException e) {
+      throw createRendererException(
+          e, formatHolder.format, PlaybackException.ERROR_CODE_DECODER_INIT_FAILED);
+    }
+    return true;
+  }
+
+  private boolean ensureEncoderConfigured() throws ExoPlaybackException {
+    if (encoder != null) {
+      return true;
+    }
+
+    try {
+      encoder =
+          MediaCodecAdapterWrapper.createForVideoEncoding(
+              encoderConfigurationOutputFormat, surface);
+    } catch (IOException e) {
+      throw createRendererException(
+          // TODO(claincly): should be "ENCODER_INIT_FAILED"
+          e,
+          checkNotNull(this.decoder).getOutputFormat(),
+          PlaybackException.ERROR_CODE_DECODER_INIT_FAILED);
+    }
+    return true;
+  }
+
+  private boolean feedDecoderFromInput() {
+    MediaCodecAdapterWrapper decoder = checkNotNull(this.decoder);
+    if (!decoder.maybeDequeueInputBuffer(buffer)) {
+      return false;
+    }
+
+    buffer.clear();
+    @SampleStream.ReadDataResult
+    int result = readSource(getFormatHolder(), buffer, /* readFlags= */ 0);
+
+    switch (result) {
+      case C.RESULT_FORMAT_READ:
+        throw new IllegalStateException("Format changes are not supported.");
+      case C.RESULT_BUFFER_READ:
+        mediaClock.updateTimeForTrackType(getTrackType(), buffer.timeUs);
+        ByteBuffer data = checkNotNull(buffer.data);
+        data.flip();
+        decoder.queueInputBuffer(buffer);
+        return !buffer.isEndOfStream();
+      case C.RESULT_NOTHING_READ:
+      default:
+        return false;
+    }
+  }
+
+  private boolean feedEncoderFromDecoder() {
+    MediaCodecAdapterWrapper decoder = checkNotNull(this.decoder);
+    if (decoder.isEnded()) {
+      return false;
+    }
+    // Rendering the decoder output queues input to the encoder because they share the same surface.
+    return decoder.maybeDequeueRenderAndReleaseOutputBuffer();
+  }
+
+  private boolean feedMuxerFromEncoder() {
+    MediaCodecAdapterWrapper encoder = checkNotNull(this.encoder);
+    if (!hasEncoderActualOutputFormat) {
+      @Nullable Format encoderOutputFormat = encoder.getOutputFormat();
+      if (encoderOutputFormat == null) {
+        return false;
+      }
+      hasEncoderActualOutputFormat = true;
+      muxerWrapper.addTrackFormat(encoderOutputFormat);
+    }
+
+    // TODO(claincly) May have to use inputStreamBuffer.isEndOfStream result to call
+    // decoder.signalEndOfInputStream().
+    MediaCodecAdapterWrapper decoder = checkNotNull(this.decoder);
+    if (decoder.isEnded()) {
+      muxerWrapper.endTrack(getTrackType());
+      muxerWrapperTrackEnded = true;
+      return false;
+    }
+
+    @Nullable ByteBuffer encoderOutputBuffer = encoder.getOutputBuffer();
+    if (encoderOutputBuffer == null) {
+      return false;
+    }
+
+    MediaCodec.BufferInfo encoderOutputBufferInfo = checkNotNull(encoder.getOutputBufferInfo());
+    if (!muxerWrapper.writeSample(
+        getTrackType(),
+        encoderOutputBuffer,
+        /* isKeyFrame= */ (encoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME) > 0,
+        encoderOutputBufferInfo.presentationTimeUs)) {
+      return false;
+    }
+    encoder.releaseOutputBuffer();
+    return true;
+  }
+}