Libvpx: Support directly outputting YUV to SurfaceView. This is supposed to use…

Libvpx: Support directly outputting YUV to SurfaceView. This is supposed to use hardware (not GPU) to convert YUV to RGB and overall use less power.

Power Comparison:

TextureView                          1080p         720p
                H264 HW              498, 496      507, 478
                VP9 RGB              1050, 1104    1185, 1152
                VP9 ANativeWindow    1070, 985     700, 674
GLSurfaceView
                VP9 YUV              1075, 1112    716, 635
SurfaceView
                H264 HW              419, 409      397, 377
                VP9 RGB              1044, 1139    654, 671
                VP9 ANativeWindow    975, 835      617, 623
                VP9 MediaCodec       683, 679      488, 476

Measures average current drawn mAH on a Nexus 6 at full brightness from time t=3 to t=95 seconds. The same clip was used for all tests. Two measurements were taken for each category.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=203847744

extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/LibvpxVideoRenderer.java

@@ -114,6 +114,7 @@ public class LibvpxVideoRenderer extends BaseRenderer {
   private final FormatHolder formatHolder;
   private final DecoderInputBuffer flagsOnlyBuffer;
   private final DrmSessionManager<ExoMediaCrypto> drmSessionManager;
+  private final boolean useSurfaceYuvOutput;
 
   private Format format;
   private VpxDecoder decoder;
@@ -177,7 +178,8 @@ public class LibvpxVideoRenderer extends BaseRenderer {
         maxDroppedFramesToNotify,
         /* drmSessionManager= */ null,
         /* playClearSamplesWithoutKeys= */ false,
-        /* disableLoopFilter= */ false);
+        /* disableLoopFilter= */ false,
+        /* useSurfaceYuvOutput= */ false);
   }
 
   /**
@@ -197,11 +199,18 @@ public class LibvpxVideoRenderer extends BaseRenderer {
    *     permitted to play clear regions of encrypted media files before {@code drmSessionManager}
    *     has obtained the keys necessary to decrypt encrypted regions of the media.
    * @param disableLoopFilter Disable the libvpx in-loop smoothing filter.
+   * @param useSurfaceYuvOutput Directly output YUV to the Surface via ANativeWindow.
    */
-  public LibvpxVideoRenderer(boolean scaleToFit, long allowedJoiningTimeMs,
-      Handler eventHandler, VideoRendererEventListener eventListener,
-      int maxDroppedFramesToNotify, DrmSessionManager<ExoMediaCrypto> drmSessionManager,
-      boolean playClearSamplesWithoutKeys, boolean disableLoopFilter) {
+  public LibvpxVideoRenderer(
+      boolean scaleToFit,
+      long allowedJoiningTimeMs,
+      Handler eventHandler,
+      VideoRendererEventListener eventListener,
+      int maxDroppedFramesToNotify,
+      DrmSessionManager<ExoMediaCrypto> drmSessionManager,
+      boolean playClearSamplesWithoutKeys,
+      boolean disableLoopFilter,
+      boolean useSurfaceYuvOutput) {
     super(C.TRACK_TYPE_VIDEO);
     this.scaleToFit = scaleToFit;
     this.disableLoopFilter = disableLoopFilter;
@@ -209,6 +218,7 @@ public class LibvpxVideoRenderer extends BaseRenderer {
     this.maxDroppedFramesToNotify = maxDroppedFramesToNotify;
     this.drmSessionManager = drmSessionManager;
     this.playClearSamplesWithoutKeys = playClearSamplesWithoutKeys;
+    this.useSurfaceYuvOutput = useSurfaceYuvOutput;
     joiningDeadlineMs = C.TIME_UNSET;
     clearReportedVideoSize();
     formatHolder = new FormatHolder();
@@ -549,21 +559,25 @@ public class LibvpxVideoRenderer extends BaseRenderer {
    *
    * @param outputBuffer The buffer to render.
    */
-  protected void renderOutputBuffer(VpxOutputBuffer outputBuffer) {
+  protected void renderOutputBuffer(VpxOutputBuffer outputBuffer) throws VpxDecoderException {
     int bufferMode = outputBuffer.mode;
     boolean renderRgb = bufferMode == VpxDecoder.OUTPUT_MODE_RGB && surface != null;
+    boolean renderSurface = bufferMode == VpxDecoder.OUTPUT_MODE_SURFACE_YUV && surface != null;
     boolean renderYuv = bufferMode == VpxDecoder.OUTPUT_MODE_YUV && outputBufferRenderer != null;
     lastRenderTimeUs = SystemClock.elapsedRealtime() * 1000;
-    if (!renderRgb && !renderYuv) {
+    if (!renderRgb && !renderYuv && !renderSurface) {
       dropOutputBuffer(outputBuffer);
     } else {
       maybeNotifyVideoSizeChanged(outputBuffer.width, outputBuffer.height);
       if (renderRgb) {
         renderRgbFrame(outputBuffer, scaleToFit);
         outputBuffer.release();
-      } else /* renderYuv */ {
+      } else if (renderYuv) {
         outputBufferRenderer.setOutputBuffer(outputBuffer);
         // The renderer will release the buffer.
+      } else { // renderSurface
+        decoder.renderToSurface(outputBuffer, surface);
+        outputBuffer.release();
       }
       consecutiveDroppedFrameCount = 0;
       decoderCounters.renderedOutputBufferCount++;
@@ -633,8 +647,13 @@ public class LibvpxVideoRenderer extends BaseRenderer {
       // The output has changed.
       this.surface = surface;
       this.outputBufferRenderer = outputBufferRenderer;
-      outputMode = outputBufferRenderer != null ? VpxDecoder.OUTPUT_MODE_YUV
-          : surface != null ? VpxDecoder.OUTPUT_MODE_RGB : VpxDecoder.OUTPUT_MODE_NONE;
+      if (surface != null) {
+        outputMode =
+            useSurfaceYuvOutput ? VpxDecoder.OUTPUT_MODE_SURFACE_YUV : VpxDecoder.OUTPUT_MODE_RGB;
+      } else {
+        outputMode =
+            outputBufferRenderer != null ? VpxDecoder.OUTPUT_MODE_YUV : VpxDecoder.OUTPUT_MODE_NONE;
+      }
       if (outputMode != VpxDecoder.OUTPUT_MODE_NONE) {
         if (decoder != null) {
           decoder.setOutputMode(outputMode);
@@ -690,7 +709,8 @@ public class LibvpxVideoRenderer extends BaseRenderer {
               NUM_OUTPUT_BUFFERS,
               INITIAL_INPUT_BUFFER_SIZE,
               mediaCrypto,
-              disableLoopFilter);
+              disableLoopFilter,
+              useSurfaceYuvOutput);
       decoder.setOutputMode(outputMode);
       TraceUtil.endSection();
       long decoderInitializedTimestamp = SystemClock.elapsedRealtime();
@@ -817,7 +837,7 @@ public class LibvpxVideoRenderer extends BaseRenderer {
    * @throws ExoPlaybackException If an error occurs processing the output buffer.
    */
   private boolean processOutputBuffer(long positionUs, long elapsedRealtimeUs)
-      throws ExoPlaybackException {
+      throws ExoPlaybackException, VpxDecoderException {
     if (initialPositionUs == C.TIME_UNSET) {
       initialPositionUs = positionUs;
     }

extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/VpxDecoder.java

@@ -15,6 +15,7 @@
  */
 package com.google.android.exoplayer2.ext.vp9;
 
+import android.view.Surface;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.decoder.CryptoInfo;
 import com.google.android.exoplayer2.decoder.SimpleDecoder;
@@ -31,6 +32,7 @@ import java.nio.ByteBuffer;
   public static final int OUTPUT_MODE_NONE = -1;
   public static final int OUTPUT_MODE_YUV = 0;
   public static final int OUTPUT_MODE_RGB = 1;
+  public static final int OUTPUT_MODE_SURFACE_YUV = 2;
 
   private static final int NO_ERROR = 0;
   private static final int DECODE_ERROR = 1;
@@ -50,10 +52,17 @@ import java.nio.ByteBuffer;
    * @param exoMediaCrypto The {@link ExoMediaCrypto} object required for decoding encrypted
    *     content. Maybe null and can be ignored if decoder does not handle encrypted content.
    * @param disableLoopFilter Disable the libvpx in-loop smoothing filter.
+   * @param enableSurfaceYuvOutputMode Whether OUTPUT_MODE_SURFACE_YUV is allowed.
    * @throws VpxDecoderException Thrown if an exception occurs when initializing the decoder.
    */
-  public VpxDecoder(int numInputBuffers, int numOutputBuffers, int initialInputBufferSize,
-      ExoMediaCrypto exoMediaCrypto, boolean disableLoopFilter) throws VpxDecoderException {
+  public VpxDecoder(
+      int numInputBuffers,
+      int numOutputBuffers,
+      int initialInputBufferSize,
+      ExoMediaCrypto exoMediaCrypto,
+      boolean disableLoopFilter,
+      boolean enableSurfaceYuvOutputMode)
+      throws VpxDecoderException {
     super(new VpxInputBuffer[numInputBuffers], new VpxOutputBuffer[numOutputBuffers]);
     if (!VpxLibrary.isAvailable()) {
       throw new VpxDecoderException("Failed to load decoder native libraries.");
@@ -62,7 +71,7 @@ import java.nio.ByteBuffer;
     if (exoMediaCrypto != null && !VpxLibrary.vpxIsSecureDecodeSupported()) {
       throw new VpxDecoderException("Vpx decoder does not support secure decode.");
     }
-    vpxDecContext = vpxInit(disableLoopFilter);
+    vpxDecContext = vpxInit(disableLoopFilter, enableSurfaceYuvOutputMode);
     if (vpxDecContext == 0) {
       throw new VpxDecoderException("Failed to initialize decoder");
     }
@@ -96,6 +105,11 @@ import java.nio.ByteBuffer;
 
   @Override
   protected void releaseOutputBuffer(VpxOutputBuffer buffer) {
+    // Decode only frames do not acquire a reference on the internal decoder buffer and thus do not
+    // require a call to vpxReleaseFrame.
+    if (outputMode == OUTPUT_MODE_SURFACE_YUV && !buffer.isDecodeOnly()) {
+      vpxReleaseFrame(vpxDecContext, buffer);
+    }
     super.releaseOutputBuffer(buffer);
   }
 
@@ -145,13 +159,36 @@ import java.nio.ByteBuffer;
     vpxClose(vpxDecContext);
   }
 
-  private native long vpxInit(boolean disableLoopFilter);
+  /** Renders the outputBuffer to the surface. Used with OUTPUT_MODE_SURFACE_YUV only. */
+  public void renderToSurface(VpxOutputBuffer outputBuffer, Surface surface)
+      throws VpxDecoderException {
+    int getFrameResult = vpxRenderFrame(vpxDecContext, surface, outputBuffer);
+    if (getFrameResult == -1) {
+      throw new VpxDecoderException("Buffer render failed.");
+    }
+  }
+
+  private native long vpxInit(boolean disableLoopFilter, boolean enableSurfaceYuvOutputMode);
+
   private native long vpxClose(long context);
   private native long vpxDecode(long context, ByteBuffer encoded, int length);
   private native long vpxSecureDecode(long context, ByteBuffer encoded, int length,
       ExoMediaCrypto mediaCrypto, int inputMode, byte[] key, byte[] iv,
       int numSubSamples, int[] numBytesOfClearData, int[] numBytesOfEncryptedData);
   private native int vpxGetFrame(long context, VpxOutputBuffer outputBuffer);
+
+  /**
+   * Renders the frame to the surface. Used with OUTPUT_MODE_SURFACE_YUV only. Must only be called
+   * if {@link #vpxInit} was called with {@code enableBufferManager = true}.
+   */
+  private native int vpxRenderFrame(long context, Surface surface, VpxOutputBuffer outputBuffer);
+
+  /**
+   * Releases the frame. Used with OUTPUT_MODE_SURFACE_YUV only. Must only be called if {@link
+   * #vpxInit} was called with {@code enableBufferManager = true}.
+   */
+  private native int vpxReleaseFrame(long context, VpxOutputBuffer outputBuffer);
+
   private native int vpxGetErrorCode(long context);
   private native String vpxGetErrorMessage(long context);
 

extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/VpxOutputBuffer.java

@@ -30,6 +30,8 @@ import java.nio.ByteBuffer;
   public static final int COLORSPACE_BT2020 = 3;
 
   private final VpxDecoder owner;
+  /** Decoder private data. */
+  public int decoderPrivate;
 
   public int mode;
   /**

extensions/vp9/src/main/jni/Android.mk

@@ -35,7 +35,7 @@ LOCAL_MODULE := libvpxJNI
 LOCAL_ARM_MODE := arm
 LOCAL_CPP_EXTENSION := .cc
 LOCAL_SRC_FILES := vpx_jni.cc
-LOCAL_LDLIBS := -llog -lz -lm
+LOCAL_LDLIBS := -llog -lz -lm -landroid
 LOCAL_SHARED_LIBRARIES := libvpx
 LOCAL_STATIC_LIBRARIES := libyuv_static cpufeatures
 include $(BUILD_SHARED_LIBRARY)

library/core/src/main/java/com/google/android/exoplayer2/decoder/SimpleDecoder.java

@@ -143,7 +143,7 @@ public abstract class SimpleDecoder<I extends DecoderInputBuffer, O extends Outp
         releaseInputBufferInternal(queuedInputBuffers.removeFirst());
       }
       while (!queuedOutputBuffers.isEmpty()) {
-        releaseOutputBufferInternal(queuedOutputBuffers.removeFirst());
+        queuedOutputBuffers.removeFirst().release();
       }
     }
   }
@@ -241,10 +241,10 @@ public abstract class SimpleDecoder<I extends DecoderInputBuffer, O extends Outp
 
     synchronized (lock) {
       if (flushed) {
-        releaseOutputBufferInternal(outputBuffer);
+        outputBuffer.release();
       } else if (outputBuffer.isDecodeOnly()) {
         skippedOutputBufferCount++;
-        releaseOutputBufferInternal(outputBuffer);
+        outputBuffer.release();
       } else {
         outputBuffer.skippedOutputBufferCount = skippedOutputBufferCount;
         skippedOutputBufferCount = 0;