Extract vpx code used for GL rendering to common classes

This will be used by both vp9 and av1 Exoplayer extensions.

PiperOrigin-RevId: 271568429

extensions/vp9/src/androidTest/java/com/google/android/exoplayer2/ext/vp9/VpxPlaybackTest.java

@@ -31,6 +31,7 @@ import com.google.android.exoplayer2.source.MediaSource;
 import com.google.android.exoplayer2.source.ProgressiveMediaSource;
 import com.google.android.exoplayer2.upstream.DefaultDataSourceFactory;
 import com.google.android.exoplayer2.util.Log;
+import com.google.android.exoplayer2.video.VideoDecoderSurfaceView;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -122,7 +123,7 @@ public class VpxPlaybackTest {
       player
           .createMessage(videoRenderer)
           .setType(LibvpxVideoRenderer.MSG_SET_OUTPUT_BUFFER_RENDERER)
-          .setPayload(new VpxVideoSurfaceView(context))
+          .setPayload(new VideoDecoderSurfaceView(context))
           .send();
       player.prepare(mediaSource);
       player.setPlayWhenReady(true);

extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/LibvpxVideoRenderer.java

@@ -35,6 +35,7 @@ import com.google.android.exoplayer2.video.SimpleDecoderVideoRenderer;
 import com.google.android.exoplayer2.video.VideoDecoderException;
 import com.google.android.exoplayer2.video.VideoDecoderInputBuffer;
 import com.google.android.exoplayer2.video.VideoDecoderOutputBuffer;
+import com.google.android.exoplayer2.video.VideoDecoderOutputBufferRenderer;
 import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
 import com.google.android.exoplayer2.video.VideoRendererEventListener;
 
@@ -48,8 +49,8 @@ import com.google.android.exoplayer2.video.VideoRendererEventListener;
  *   <li>Message with type {@link C#MSG_SET_SURFACE} to set the output surface. The message payload
  *       should be the target {@link Surface}, or null.
  *   <li>Message with type {@link #MSG_SET_OUTPUT_BUFFER_RENDERER} to set the output buffer
- *       renderer. The message payload should be the target {@link VpxOutputBufferRenderer}, or
- *       null.
+ *       renderer. The message payload should be the target {@link
+ *       VideoDecoderOutputBufferRenderer}, or null.
  * </ul>
  */
 public class LibvpxVideoRenderer extends SimpleDecoderVideoRenderer {
@@ -57,7 +58,7 @@ public class LibvpxVideoRenderer extends SimpleDecoderVideoRenderer {
   /**
    * The type of a message that can be passed to an instance of this class via {@link
    * ExoPlayer#createMessage(Target)}. The message payload should be the target {@link
-   * VpxOutputBufferRenderer}, or null.
+   * VideoDecoderOutputBufferRenderer}, or null.
    */
   public static final int MSG_SET_OUTPUT_BUFFER_RENDERER = C.MSG_CUSTOM_BASE;
 
@@ -79,11 +80,11 @@ public class LibvpxVideoRenderer extends SimpleDecoderVideoRenderer {
   private final int threads;
 
   private Surface surface;
-  private VpxOutputBufferRenderer outputBufferRenderer;
+  private VideoDecoderOutputBufferRenderer outputBufferRenderer;
   @C.VideoOutputMode private int outputMode;
 
   private VpxDecoder decoder;
-  private VpxOutputBuffer outputBuffer;
+  private VideoDecoderOutputBuffer outputBuffer;
 
   private VideoFrameMetadataListener frameMetadataListener;
 
@@ -298,7 +299,7 @@ public class LibvpxVideoRenderer extends SimpleDecoderVideoRenderer {
     if (messageType == C.MSG_SET_SURFACE) {
       setOutput((Surface) message, null);
     } else if (messageType == MSG_SET_OUTPUT_BUFFER_RENDERER) {
-      setOutput(null, (VpxOutputBufferRenderer) message);
+      setOutput(null, (VideoDecoderOutputBufferRenderer) message);
     } else if (messageType == C.MSG_SET_VIDEO_FRAME_METADATA_LISTENER) {
       frameMetadataListener = (VideoFrameMetadataListener) message;
     } else {
@@ -309,7 +310,7 @@ public class LibvpxVideoRenderer extends SimpleDecoderVideoRenderer {
   // Internal methods.
 
   private void setOutput(
-      @Nullable Surface surface, @Nullable VpxOutputBufferRenderer outputBufferRenderer) {
+      @Nullable Surface surface, @Nullable VideoDecoderOutputBufferRenderer outputBufferRenderer) {
     // At most one output may be non-null. Both may be null if the output is being cleared.
     Assertions.checkState(surface == null || outputBufferRenderer == null);
     if (this.surface != surface || this.outputBufferRenderer != outputBufferRenderer) {

extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/VpxDecoder.java

@@ -24,11 +24,12 @@ import com.google.android.exoplayer2.drm.DecryptionException;
 import com.google.android.exoplayer2.drm.ExoMediaCrypto;
 import com.google.android.exoplayer2.util.Util;
 import com.google.android.exoplayer2.video.VideoDecoderInputBuffer;
+import com.google.android.exoplayer2.video.VideoDecoderOutputBuffer;
 import java.nio.ByteBuffer;
 
 /** Vpx decoder. */
 /* package */ final class VpxDecoder
-    extends SimpleDecoder<VideoDecoderInputBuffer, VpxOutputBuffer, VpxDecoderException> {
+    extends SimpleDecoder<VideoDecoderInputBuffer, VideoDecoderOutputBuffer, VpxDecoderException> {
 
   // These constants should match the codes returned from vpxDecode and vpxSecureDecode functions in
   // https://github.com/google/ExoPlayer/blob/release-v2/extensions/vp9/src/main/jni/vpx_jni.cc.
@@ -63,7 +64,9 @@ import java.nio.ByteBuffer;
       boolean enableRowMultiThreadMode,
       int threads)
       throws VpxDecoderException {
-    super(new VideoDecoderInputBuffer[numInputBuffers], new VpxOutputBuffer[numOutputBuffers]);
+    super(
+        new VideoDecoderInputBuffer[numInputBuffers],
+        new VideoDecoderOutputBuffer[numOutputBuffers]);
     if (!VpxLibrary.isAvailable()) {
       throw new VpxDecoderException("Failed to load decoder native libraries.");
     }
@@ -98,12 +101,12 @@ import java.nio.ByteBuffer;
   }
 
   @Override
-  protected VpxOutputBuffer createOutputBuffer() {
-    return new VpxOutputBuffer(this);
+  protected VideoDecoderOutputBuffer createOutputBuffer() {
+    return new VideoDecoderOutputBuffer(this::releaseOutputBuffer);
   }
 
   @Override
-  protected void releaseOutputBuffer(VpxOutputBuffer buffer) {
+  protected void releaseOutputBuffer(VideoDecoderOutputBuffer buffer) {
     // Decode only frames do not acquire a reference on the internal decoder buffer and thus do not
     // require a call to vpxReleaseFrame.
     if (outputMode == C.VIDEO_OUTPUT_MODE_SURFACE_YUV && !buffer.isDecodeOnly()) {
@@ -120,7 +123,7 @@ import java.nio.ByteBuffer;
   @Override
   @Nullable
   protected VpxDecoderException decode(
-      VideoDecoderInputBuffer inputBuffer, VpxOutputBuffer outputBuffer, boolean reset) {
+      VideoDecoderInputBuffer inputBuffer, VideoDecoderOutputBuffer outputBuffer, boolean reset) {
     ByteBuffer inputData = Util.castNonNull(inputBuffer.data);
     int inputSize = inputData.limit();
     CryptoInfo cryptoInfo = inputBuffer.cryptoInfo;
@@ -163,7 +166,7 @@ import java.nio.ByteBuffer;
   }
 
   /** Renders the outputBuffer to the surface. Used with OUTPUT_MODE_SURFACE_YUV only. */
-  public void renderToSurface(VpxOutputBuffer outputBuffer, Surface surface)
+  public void renderToSurface(VideoDecoderOutputBuffer outputBuffer, Surface surface)
       throws VpxDecoderException {
     int getFrameResult = vpxRenderFrame(vpxDecContext, surface, outputBuffer);
     if (getFrameResult == -1) {
@@ -189,19 +192,20 @@ import java.nio.ByteBuffer;
       int[] numBytesOfClearData,
       int[] numBytesOfEncryptedData);
 
-  private native int vpxGetFrame(long context, VpxOutputBuffer outputBuffer);
+  private native int vpxGetFrame(long context, VideoDecoderOutputBuffer outputBuffer);
 
   /**
    * Renders the frame to the surface. Used with OUTPUT_MODE_SURFACE_YUV only. Must only be called
    * if {@link #vpxInit} was called with {@code enableBufferManager = true}.
    */
-  private native int vpxRenderFrame(long context, Surface surface, VpxOutputBuffer outputBuffer);
+  private native int vpxRenderFrame(
+      long context, Surface surface, VideoDecoderOutputBuffer outputBuffer);
 
   /**
    * Releases the frame. Used with OUTPUT_MODE_SURFACE_YUV only. Must only be called if {@link
    * #vpxInit} was called with {@code enableBufferManager = true}.
    */
-  private native int vpxReleaseFrame(long context, VpxOutputBuffer outputBuffer);
+  private native int vpxReleaseFrame(long context, VideoDecoderOutputBuffer outputBuffer);
 
   private native int vpxGetErrorCode(long context);
   private native String vpxGetErrorMessage(long context);

extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/VpxOutputBuffer.java

@@ -17,18 +17,22 @@ package com.google.android.exoplayer2.ext.vp9;
 
 import com.google.android.exoplayer2.video.VideoDecoderOutputBuffer;
 
-/** Video output buffer, populated by {@link VpxDecoder}. */
+// TODO(b/139174707): Delete this class once binaries in WVVp9OpusPlaybackTest are updated to depend
+// on VideoDecoderOutputBuffer. Also mark VideoDecoderOutputBuffer as final.
+/**
+ * Video output buffer, populated by {@link VpxDecoder}.
+ *
+ * @deprecated Use {@link VideoDecoderOutputBuffer} instead.
+ */
+@Deprecated
 public final class VpxOutputBuffer extends VideoDecoderOutputBuffer {
 
-  private final VpxDecoder owner;
-
-  public VpxOutputBuffer(VpxDecoder owner) {
-    this.owner = owner;
+  /**
+   * Creates VpxOutputBuffer.
+   *
+   * @param owner Buffer owner.
+   */
+  public VpxOutputBuffer(VideoDecoderOutputBuffer.Owner owner) {
+    super(owner);
   }
-
-  @Override
-  public void release() {
-    owner.releaseOutputBuffer(this);
-  }
-
 }

extensions/vp9/src/main/jni/vpx_jni.cc

@@ -38,27 +38,27 @@
 #define LOGE(...) ((void)__android_log_print(ANDROID_LOG_ERROR, LOG_TAG, \
                                              __VA_ARGS__))
 
-#define DECODER_FUNC(RETURN_TYPE, NAME, ...) \
-  extern "C" { \
-  JNIEXPORT RETURN_TYPE \
-    Java_com_google_android_exoplayer2_ext_vp9_VpxDecoder_ ## NAME \
-      (JNIEnv* env, jobject thiz, ##__VA_ARGS__);\
-  } \
-  JNIEXPORT RETURN_TYPE \
-    Java_com_google_android_exoplayer2_ext_vp9_VpxDecoder_ ## NAME \
-      (JNIEnv* env, jobject thiz, ##__VA_ARGS__)\
-
-#define LIBRARY_FUNC(RETURN_TYPE, NAME, ...) \
-  extern "C" { \
-  JNIEXPORT RETURN_TYPE \
-    Java_com_google_android_exoplayer2_ext_vp9_VpxLibrary_ ## NAME \
-      (JNIEnv* env, jobject thiz, ##__VA_ARGS__);\
-  } \
-  JNIEXPORT RETURN_TYPE \
-    Java_com_google_android_exoplayer2_ext_vp9_VpxLibrary_ ## NAME \
-      (JNIEnv* env, jobject thiz, ##__VA_ARGS__)\
-
-// JNI references for VpxOutputBuffer class.
+#define DECODER_FUNC(RETURN_TYPE, NAME, ...)                        \
+  extern "C" {                                                      \
+  JNIEXPORT RETURN_TYPE                                             \
+      Java_com_google_android_exoplayer2_ext_vp9_VpxDecoder_##NAME( \
+          JNIEnv* env, jobject thiz, ##__VA_ARGS__);                \
+  }                                                                 \
+  JNIEXPORT RETURN_TYPE                                             \
+      Java_com_google_android_exoplayer2_ext_vp9_VpxDecoder_##NAME( \
+          JNIEnv* env, jobject thiz, ##__VA_ARGS__)
+
+#define LIBRARY_FUNC(RETURN_TYPE, NAME, ...)                        \
+  extern "C" {                                                      \
+  JNIEXPORT RETURN_TYPE                                             \
+      Java_com_google_android_exoplayer2_ext_vp9_VpxLibrary_##NAME( \
+          JNIEnv* env, jobject thiz, ##__VA_ARGS__);                \
+  }                                                                 \
+  JNIEXPORT RETURN_TYPE                                             \
+      Java_com_google_android_exoplayer2_ext_vp9_VpxLibrary_##NAME( \
+          JNIEnv* env, jobject thiz, ##__VA_ARGS__)
+
+// JNI references for VideoDecoderOutputBuffer class.
 static jmethodID initForYuvFrame;
 static jmethodID initForPrivateFrame;
 static jfieldID dataField;
@@ -477,7 +477,7 @@ DECODER_FUNC(jlong, vpxInit, jboolean disableLoopFilter,
 
   // Populate JNI References.
   const jclass outputBufferClass = env->FindClass(
-      "com/google/android/exoplayer2/ext/vp9/VpxOutputBuffer");
+      "com/google/android/exoplayer2/video/VideoDecoderOutputBuffer");
   initForYuvFrame = env->GetMethodID(outputBufferClass, "initForYuvFrame",
                                      "(IIIII)Z");
   initForPrivateFrame =

library/core/proguard-rules.txt

@@ -61,3 +61,8 @@
 # Don't warn about checkerframework and Kotlin annotations
 -dontwarn org.checkerframework.**
 -dontwarn kotlin.annotations.jvm.**
+
+# Some members of this class are being accessed from native methods. Keep them unobfuscated.
+-keep class com.google.android.exoplayer2.ext.video.VideoDecoderOutputBuffer {
+    *;
+}

library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderOutputBuffer.java

@@ -21,7 +21,18 @@ import com.google.android.exoplayer2.decoder.OutputBuffer;
 import java.nio.ByteBuffer;
 
 /** Video decoder output buffer containing video frame data. */
-public abstract class VideoDecoderOutputBuffer extends OutputBuffer {
+public class VideoDecoderOutputBuffer extends OutputBuffer {
+
+  /** Buffer owner. */
+  public interface Owner {
+
+    /**
+     * Releases the buffer.
+     *
+     * @param outputBuffer Output buffer.
+     */
+    public void releaseOutputBuffer(VideoDecoderOutputBuffer outputBuffer);
+  }
 
   // LINT.IfChange
   public static final int COLORSPACE_UNKNOWN = 0;
@@ -54,6 +65,22 @@ public abstract class VideoDecoderOutputBuffer extends OutputBuffer {
    */
   @Nullable public ByteBuffer supplementalData;
 
+  private final Owner owner;
+
+  /**
+   * Creates VideoDecoderOutputBuffer.
+   *
+   * @param owner Buffer owner.
+   */
+  public VideoDecoderOutputBuffer(Owner owner) {
+    this.owner = owner;
+  }
+
+  @Override
+  public void release() {
+    owner.releaseOutputBuffer(this);
+  }
+
   /**
    * Initializes the buffer.
    *

extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/VpxOutputBufferRenderer.java → library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderOutputBufferRenderer.java

@@ -13,18 +13,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.google.android.exoplayer2.ext.vp9;
+package com.google.android.exoplayer2.video;
 
-/**
- * Renders the {@link VpxOutputBuffer}.
- */
-public interface VpxOutputBufferRenderer {
+/** Renders the {@link VideoDecoderOutputBuffer}. */
+public interface VideoDecoderOutputBufferRenderer {
 
   /**
    * Sets the output buffer to be rendered. The renderer is responsible for releasing the buffer.
    *
    * @param outputBuffer The output buffer to be rendered.
    */
-  void setOutputBuffer(VpxOutputBuffer outputBuffer);
-
+  void setOutputBuffer(VideoDecoderOutputBuffer outputBuffer);
 }

extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/VpxRenderer.java → library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderRenderer.java

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.google.android.exoplayer2.ext.vp9;
+package com.google.android.exoplayer2.video;
 
 import android.opengl.GLES20;
 import android.opengl.GLSurfaceView;
@@ -24,10 +24,10 @@ import javax.microedition.khronos.egl.EGLConfig;
 import javax.microedition.khronos.opengles.GL10;
 
 /**
- * GLSurfaceView.Renderer implementation that can render YUV Frames returned by libvpx after
- * decoding. It does the YUV to RGB color conversion in the Fragment Shader.
+ * GLSurfaceView.Renderer implementation that can render YUV Frames returned by a video decoder
+ * after decoding. It does the YUV to RGB color conversion in the Fragment Shader.
  */
-/* package */ class VpxRenderer implements GLSurfaceView.Renderer {
+/* package */ class VideoDecoderRenderer implements GLSurfaceView.Renderer {
 
   private static final float[] kColorConversion601 = {
     1.164f, 1.164f, 1.164f,
@@ -74,7 +74,7 @@ import javax.microedition.khronos.opengles.GL10;
   private static final FloatBuffer TEXTURE_VERTICES =
       GlUtil.createBuffer(new float[] {-1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f});
   private final int[] yuvTextures = new int[3];
-  private final AtomicReference<VpxOutputBuffer> pendingOutputBufferReference;
+  private final AtomicReference<VideoDecoderOutputBuffer> pendingOutputBufferReference;
 
   // Kept in a field rather than a local variable so that it doesn't get garbage collected before
   // glDrawArrays uses it.
@@ -86,9 +86,9 @@ import javax.microedition.khronos.opengles.GL10;
   private int previousWidth;
   private int previousStride;
 
-  private VpxOutputBuffer renderedOutputBuffer; // Accessed only from the GL thread.
+  private VideoDecoderOutputBuffer renderedOutputBuffer; // Accessed only from the GL thread.
 
-  public VpxRenderer() {
+  public VideoDecoderRenderer() {
     previousWidth = -1;
     previousStride = -1;
     pendingOutputBufferReference = new AtomicReference<>();
@@ -96,12 +96,13 @@ import javax.microedition.khronos.opengles.GL10;
 
   /**
    * Set a frame to be rendered. This should be followed by a call to
-   * VpxVideoSurfaceView.requestRender() to actually render the frame.
+   * VideoDecoderSurfaceView.requestRender() to actually render the frame.
    *
    * @param outputBuffer OutputBuffer containing the YUV Frame to be rendered
    */
-  public void setFrame(VpxOutputBuffer outputBuffer) {
-    VpxOutputBuffer oldPendingOutputBuffer = pendingOutputBufferReference.getAndSet(outputBuffer);
+  public void setFrame(VideoDecoderOutputBuffer outputBuffer) {
+    VideoDecoderOutputBuffer oldPendingOutputBuffer =
+        pendingOutputBufferReference.getAndSet(outputBuffer);
     if (oldPendingOutputBuffer != null) {
       // The old pending output buffer will never be used for rendering, so release it now.
       oldPendingOutputBuffer.release();
@@ -132,7 +133,7 @@ import javax.microedition.khronos.opengles.GL10;
 
   @Override
   public void onDrawFrame(GL10 unused) {
-    VpxOutputBuffer pendingOutputBuffer = pendingOutputBufferReference.getAndSet(null);
+    VideoDecoderOutputBuffer pendingOutputBuffer = pendingOutputBufferReference.getAndSet(null);
     if (pendingOutputBuffer == null && renderedOutputBuffer == null) {
       // There is no output buffer to render at the moment.
       return;
@@ -143,17 +144,17 @@ import javax.microedition.khronos.opengles.GL10;
       }
       renderedOutputBuffer = pendingOutputBuffer;
     }
-    VpxOutputBuffer outputBuffer = renderedOutputBuffer;
+    VideoDecoderOutputBuffer outputBuffer = renderedOutputBuffer;
     // Set color matrix. Assume BT709 if the color space is unknown.
     float[] colorConversion = kColorConversion709;
     switch (outputBuffer.colorspace) {
-      case VpxOutputBuffer.COLORSPACE_BT601:
+      case VideoDecoderOutputBuffer.COLORSPACE_BT601:
         colorConversion = kColorConversion601;
         break;
-      case VpxOutputBuffer.COLORSPACE_BT2020:
+      case VideoDecoderOutputBuffer.COLORSPACE_BT2020:
         colorConversion = kColorConversion2020;
         break;
-      case VpxOutputBuffer.COLORSPACE_BT709:
+      case VideoDecoderOutputBuffer.COLORSPACE_BT709:
       default:
         break; // Do nothing
     }

extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/VpxVideoSurfaceView.java → library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderSurfaceView.java

@@ -13,27 +13,26 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.google.android.exoplayer2.ext.vp9;
+package com.google.android.exoplayer2.video;
 
 import android.content.Context;
 import android.opengl.GLSurfaceView;
 import android.util.AttributeSet;
 import androidx.annotation.Nullable;
 
-/**
- * A GLSurfaceView extension that scales itself to the given aspect ratio.
- */
-public class VpxVideoSurfaceView extends GLSurfaceView implements VpxOutputBufferRenderer {
+/** A GLSurfaceView extension that scales itself to the given aspect ratio. */
+public class VideoDecoderSurfaceView extends GLSurfaceView
+    implements VideoDecoderOutputBufferRenderer {
 
-  private final VpxRenderer renderer;
+  private final VideoDecoderRenderer renderer;
 
-  public VpxVideoSurfaceView(Context context) {
+  public VideoDecoderSurfaceView(Context context) {
     this(context, /* attrs= */ null);
   }
 
-  public VpxVideoSurfaceView(Context context, @Nullable AttributeSet attrs) {
+  public VideoDecoderSurfaceView(Context context, @Nullable AttributeSet attrs) {
     super(context, attrs);
-    renderer = new VpxRenderer();
+    renderer = new VideoDecoderRenderer();
     setPreserveEGLContextOnPause(true);
     setEGLContextClientVersion(2);
     setRenderer(renderer);
@@ -41,7 +40,7 @@ public class VpxVideoSurfaceView extends GLSurfaceView implements VpxOutputBuffe
   }
 
   @Override
-  public void setOutputBuffer(VpxOutputBuffer outputBuffer) {
+  public void setOutputBuffer(VideoDecoderOutputBuffer outputBuffer) {
     renderer.setFrame(outputBuffer);
     requestRender();
   }