Support GL rendering with SimpleExoPlayer and PlayerView

PiperOrigin-RevId: 273760294

Support GL rendering with SimpleExoPlayer and PlayerView
PiperOrigin-RevId: 273760294
5cf82a50 · sofijajvc · Ian Baker · a35a0925 · 5cf82a50 · 5cf82a50
Commit 5cf82a50 authored Oct 09, 2019 by sofijajvc Committed by Ian Baker Oct 10, 2019
Showing with 132 additions and 66 deletions
extensions/av1/README.md
extensions/vp9/README.md
extensions/vp9/src/androidTest/java/com/google/android/exoplayer2/ext/vp9/VpxPlaybackTest.java
library/core/src/main/java/com/google/android/exoplayer2/Player.java
library/core/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java
library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderRenderer.java
library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderSurfaceView.java
library/ui/src/main/java/com/google/android/exoplayer2/ui/PlayerView.java
library/ui/src/main/res/values/attrs.xml
--- a/extensions/av1/README.md
+++ b/extensions/av1/README.md
@@ -79,23 +79,26 @@ a custom track selector the choice of `Renderer` is up to your implementation.
 You need to make sure you are passing a `Libgav1VideoRenderer` to the player and
 then you need to implement your own logic to use the renderer for a given track.
+## Rendering options ##
 There are two possibilities for rendering the output `Libgav1VideoRenderer`
 gets from the libgav1 decoder:
-* Native rendering with `ANativeWindow`
+* GL rendering using GL shader for color space conversion
-* OpenGL rendering.
+  * If you are using `SimpleExoPlayer` with `PlayerView`, enable this option by
+    setting `surface_type` of `PlayerView` to be `video_decoder_surface_view`.
+  * Otherwise, enable this option by sending `Libgav1VideoRenderer` a message
+    of type `C.MSG_SET_OUTPUT_BUFFER_RENDERER` with an instance of
+    `VideoDecoderOutputBufferRenderer` as its object.
-`SimpleExoPlayer` uses `ANativeWindow` rendering. To enable this mode send the
+* Native rendering using `ANativeWindow`
-renderer a message of type `C.MSG_SET_SURFACE` with a `Surface` as its object.
+  * If you are using `SimpleExoPlayer` with `PlayerView`, this option is enabled
-`Libgav1VideoRenderer` can also output to a `VideoDecoderSurfaceView` when
+    by default.
-not being used via `SimpleExoPlayer`, in which case color space conversion will
+  * Otherwise, enable this option by sending `Libgav1VideoRenderer` a message of
-be performed using a GL shader. To enable this mode, send the renderer a message
+    type `C.MSG_SET_SURFACE` with an instance of `SurfaceView` as its object.
-of type `C.MSG_SET_OUTPUT_BUFFER_RENDERER` with the `VideoDecoderSurfaceView` as
-its object.
 Note: Although the default option uses `ANativeWindow`, based on our testing the
-GL rendering mode has better performance, so should be preferred by apps that
+GL rendering mode has better performance, so should be preferred
-can use `VideoDecoderSurfaceView`.
 ## Links ##

--- a/extensions/vp9/README.md
+++ b/extensions/vp9/README.md
@@ -107,11 +107,26 @@ a custom track selector the choice of `Renderer` is up to your implementation,
 so you need to make sure you are passing an `LibvpxVideoRenderer` to the
 player, then implement your own logic to use the renderer for a given track.
-`LibvpxVideoRenderer` can optionally output to a `VideoDecoderSurfaceView` when
+## Rendering options ##
-not being used via `SimpleExoPlayer`, in which case color space conversion will
-be performed using a GL shader. To enable this mode, send the renderer a message
+There are two possibilities for rendering the output `LibvpxVideoRenderer`
-of type `C.MSG_SET_OUTPUT_BUFFER_RENDERER` with the `VideoDecoderSurfaceView` as
+gets from the libvpx decoder:
-its object, instead of sending `MSG_SET_SURFACE` with a `Surface`.
+* GL rendering using GL shader for color space conversion
+  * If you are using `SimpleExoPlayer` with `PlayerView`, enable this option by
+    setting `surface_type` of `PlayerView` to be `video_decoder_surface_view`.
+  * Otherwise, enable this option by sending `LibvpxVideoRenderer` a message of
+    type `C.MSG_SET_OUTPUT_BUFFER_RENDERER` with an instance of
+    `VideoDecoderOutputBufferRenderer` as its object.
+* Native rendering using `ANativeWindow`
+  * If you are using `SimpleExoPlayer` with `PlayerView`, this option is enabled
+    by default.
+  * Otherwise, enable this option by sending `LibvpxVideoRenderer` a message of
+    type `C.MSG_SET_SURFACE` with an instance of `SurfaceView` as its object.
+Note: Although the default option uses `ANativeWindow`, based on our testing the
+GL rendering mode has better performance, so should be preferred.
 ## Links ##

--- a/extensions/vp9/src/androidTest/java/com/google/android/exoplayer2/ext/vp9/VpxPlaybackTest.java
+++ b/extensions/vp9/src/androidTest/java/com/google/android/exoplayer2/ext/vp9/VpxPlaybackTest.java
@@ -124,7 +124,7 @@ public class VpxPlaybackTest {
      player
          .createMessage(videoRenderer)
          .setType(C.MSG_SET_OUTPUT_BUFFER_RENDERER)
-          .setPayload(new VideoDecoderSurfaceView(context))
+          .setPayload(new VideoDecoderSurfaceView(context).getOutputBufferRenderer())
          .send();
      player.prepare(mediaSource);
      player.setPlayWhenReady(true);

--- a/library/core/src/main/java/com/google/android/exoplayer2/Player.java
+++ b/library/core/src/main/java/com/google/android/exoplayer2/Player.java
@@ -31,6 +31,7 @@ import com.google.android.exoplayer2.source.TrackGroupArray;
 import com.google.android.exoplayer2.text.TextOutput;
 import com.google.android.exoplayer2.trackselection.TrackSelectionArray;
 import com.google.android.exoplayer2.util.Util;
+import com.google.android.exoplayer2.video.VideoDecoderOutputBufferRenderer;
 import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
 import com.google.android.exoplayer2.video.VideoListener;
 import com.google.android.exoplayer2.video.spherical.CameraMotionListener;
@@ -280,6 +281,13 @@ public interface Player {
     * @param textureView The texture view to clear.
     */
    void clearVideoTextureView(TextureView textureView);
+    /**
+     * Sets the output buffer renderer.
+     *
+     * @param outputBufferRenderer The output buffer renderer.
+     */
+    void setOutputBufferRenderer(VideoDecoderOutputBufferRenderer outputBufferRenderer);
  }
  /** The text component of a {@link Player}. */

--- a/library/core/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java
+++ b/library/core/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java
@@ -55,6 +55,7 @@ import com.google.android.exoplayer2.util.Clock;
 import com.google.android.exoplayer2.util.Log;
 import com.google.android.exoplayer2.util.PriorityTaskManager;
 import com.google.android.exoplayer2.util.Util;
+import com.google.android.exoplayer2.video.VideoDecoderOutputBufferRenderer;
 import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
 import com.google.android.exoplayer2.video.VideoRendererEventListener;
 import com.google.android.exoplayer2.video.spherical.CameraMotionListener;
@@ -584,8 +585,8 @@ public class SimpleExoPlayer extends BasePlayer
        Log.w(TAG, "Replacing existing SurfaceTextureListener.");
      }
      textureView.setSurfaceTextureListener(componentListener);
-      SurfaceTexture surfaceTexture = textureView.isAvailable() ? textureView.getSurfaceTexture()
+      SurfaceTexture surfaceTexture =
-          : null;
+          textureView.isAvailable() ? textureView.getSurfaceTexture() : null;
      if (surfaceTexture == null) {
        setVideoSurfaceInternal(/* surface= */ null, /* ownsSurface= */ true);
        maybeNotifySurfaceSizeChanged(/* width= */ 0, /* height= */ 0);
@@ -605,6 +606,23 @@ public class SimpleExoPlayer extends BasePlayer
  }
  @Override
+  public void setOutputBufferRenderer(VideoDecoderOutputBufferRenderer outputBufferRenderer) {
+    verifyApplicationThread();
+    removeSurfaceCallbacks();
+    List<PlayerMessage> messages = new ArrayList<>();
+    for (Renderer renderer : renderers) {
+      if (renderer.getTrackType() == C.TRACK_TYPE_VIDEO) {
+        messages.add(
+            player
+                .createMessage(renderer)
+                .setType(C.MSG_SET_OUTPUT_BUFFER_RENDERER)
+                .setPayload(outputBufferRenderer)
+                .send());
+      }
+    }
+  }
+  @Override
  public void addAudioListener(AudioListener listener) {
    audioListeners.add(listener);
  }
@@ -695,12 +713,12 @@ public class SimpleExoPlayer extends BasePlayer
  /**
   * Sets the stream type for audio playback, used by the underlying audio track.
-   * <p>
+   *
-   * Setting the stream type during playback may introduce a short gap in audio output as the audio
+   * <p>Setting the stream type during playback may introduce a short gap in audio output as the
-   * track is recreated. A new audio session id will also be generated.
+   * audio track is recreated. A new audio session id will also be generated.
-   * <p>
+   *
-   * Calling this method overwrites any attributes set previously by calling
+   * <p>Calling this method overwrites any attributes set previously by calling {@link
-   * {@link #setAudioAttributes(AudioAttributes)}.
+   * #setAudioAttributes(AudioAttributes)}.
   *
   * @deprecated Use {@link #setAudioAttributes(AudioAttributes)}.
   * @param streamType The stream type for audio playback.
@@ -1473,11 +1491,11 @@ public class SimpleExoPlayer extends BasePlayer
    }
    @Override
-    public void onVideoDecoderInitialized(String decoderName, long initializedTimestampMs,
+    public void onVideoDecoderInitialized(
-        long initializationDurationMs) {
+        String decoderName, long initializedTimestampMs, long initializationDurationMs) {
      for (VideoRendererEventListener videoDebugListener : videoDebugListeners) {
-        videoDebugListener.onVideoDecoderInitialized(decoderName, initializedTimestampMs,
+        videoDebugListener.onVideoDecoderInitialized(
-            initializationDurationMs);
+            decoderName, initializedTimestampMs, initializationDurationMs);
      }
    }
@@ -1497,8 +1515,8 @@ public class SimpleExoPlayer extends BasePlayer
    }
    @Override
-    public void onVideoSizeChanged(int width, int height, int unappliedRotationDegrees,
+    public void onVideoSizeChanged(
-        float pixelWidthHeightRatio) {
+        int width, int height, int unappliedRotationDegrees, float pixelWidthHeightRatio) {
      for (com.google.android.exoplayer2.video.VideoListener videoListener : videoListeners) {
        // Prevent duplicate notification if a listener is both a VideoRendererEventListener and
        // a VideoListener, as they have the same method signature.
@@ -1508,8 +1526,8 @@ public class SimpleExoPlayer extends BasePlayer
        }
      }
      for (VideoRendererEventListener videoDebugListener : videoDebugListeners) {
-        videoDebugListener.onVideoSizeChanged(width, height, unappliedRotationDegrees,
+        videoDebugListener.onVideoSizeChanged(
-            pixelWidthHeightRatio);
+            width, height, unappliedRotationDegrees, pixelWidthHeightRatio);
      }
    }
@@ -1563,11 +1581,11 @@ public class SimpleExoPlayer extends BasePlayer
    }
    @Override
-    public void onAudioDecoderInitialized(String decoderName, long initializedTimestampMs,
+    public void onAudioDecoderInitialized(
-        long initializationDurationMs) {
+        String decoderName, long initializedTimestampMs, long initializationDurationMs) {
      for (AudioRendererEventListener audioDebugListener : audioDebugListeners) {
-        audioDebugListener.onAudioDecoderInitialized(decoderName, initializedTimestampMs,
+        audioDebugListener.onAudioDecoderInitialized(
-            initializationDurationMs);
+            decoderName, initializedTimestampMs, initializationDurationMs);
      }
    }
@@ -1580,8 +1598,8 @@ public class SimpleExoPlayer extends BasePlayer
    }
    @Override
-    public void onAudioSinkUnderrun(int bufferSize, long bufferSizeMs,
+    public void onAudioSinkUnderrun(
-        long elapsedSinceLastFeedMs) {
+        int bufferSize, long bufferSizeMs, long elapsedSinceLastFeedMs) {
      for (AudioRendererEventListener audioDebugListener : audioDebugListeners) {
        audioDebugListener.onAudioSinkUnderrun(bufferSize, bufferSizeMs, elapsedSinceLastFeedMs);
      }

--- a/library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderRenderer.java
+++ b/library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderRenderer.java
@@ -28,7 +28,8 @@ import javax.microedition.khronos.opengles.GL10;
 * GLSurfaceView.Renderer implementation that can render YUV Frames returned by a video decoder
 * after decoding. It does the YUV to RGB color conversion in the Fragment Shader.
 */
-/* package */ class VideoDecoderRenderer implements GLSurfaceView.Renderer {
+/* package */ class VideoDecoderRenderer
+    implements GLSurfaceView.Renderer, VideoDecoderOutputBufferRenderer {
  private static final float[] kColorConversion601 = {
    1.164f, 1.164f, 1.164f,
@@ -82,6 +83,7 @@ import javax.microedition.khronos.opengles.GL10;
  private static final FloatBuffer TEXTURE_VERTICES =
      GlUtil.createBuffer(new float[] {-1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f});
+  private final GLSurfaceView surfaceView;
  private final int[] yuvTextures = new int[3];
  private final AtomicReference<VideoDecoderOutputBuffer> pendingOutputBufferReference;
@@ -98,7 +100,8 @@ import javax.microedition.khronos.opengles.GL10;
  private VideoDecoderOutputBuffer renderedOutputBuffer; // Accessed only from the GL thread.
-  public VideoDecoderRenderer() {
+  public VideoDecoderRenderer(GLSurfaceView surfaceView) {
+    this.surfaceView = surfaceView;
    pendingOutputBufferReference = new AtomicReference<>();
    textureCoords = new FloatBuffer[3];
    texLocations = new int[3];
@@ -109,21 +112,6 @@ import javax.microedition.khronos.opengles.GL10;
    }
  }
-  /**
-   * Set a frame to be rendered. This should be followed by a call to
-   * VideoDecoderSurfaceView.requestRender() to actually render the frame.
-   *
-   * @param outputBuffer OutputBuffer containing the YUV Frame to be rendered
-   */
-  public void setFrame(VideoDecoderOutputBuffer outputBuffer) {
-    VideoDecoderOutputBuffer oldPendingOutputBuffer =
-        pendingOutputBufferReference.getAndSet(outputBuffer);
-    if (oldPendingOutputBuffer != null) {
-      // The old pending output buffer will never be used for rendering, so release it now.
-      oldPendingOutputBuffer.release();
-    }
-  }
  @Override
  public void onSurfaceCreated(GL10 unused, EGLConfig config) {
    program = GlUtil.compileProgram(VERTEX_SHADER, FRAGMENT_SHADER);
@@ -223,6 +211,17 @@ import javax.microedition.khronos.opengles.GL10;
    GlUtil.checkGlError();
  }
+  @Override
+  public void setOutputBuffer(VideoDecoderOutputBuffer outputBuffer) {
+    VideoDecoderOutputBuffer oldPendingOutputBuffer =
+        pendingOutputBufferReference.getAndSet(outputBuffer);
+    if (oldPendingOutputBuffer != null) {
+      // The old pending output buffer will never be used for rendering, so release it now.
+      oldPendingOutputBuffer.release();
+    }
+    surfaceView.requestRender();
+  }
  private void setupTextures() {
    GLES20.glGenTextures(3, yuvTextures, 0);
    for (int i = 0; i < 3; i++) {

--- a/library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderSurfaceView.java
+++ b/library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderSurfaceView.java
@@ -21,28 +21,40 @@ import android.util.AttributeSet;
 import androidx.annotation.Nullable;
 /** A GLSurfaceView extension that scales itself to the given aspect ratio. */
-public class VideoDecoderSurfaceView extends GLSurfaceView
+public class VideoDecoderSurfaceView extends GLSurfaceView {
-    implements VideoDecoderOutputBufferRenderer {
  private final VideoDecoderRenderer renderer;
+  /**
+   * Creates VideoDecoderSurfaceView.
+   *
+   * @param context A {@link Context}.
+   */
  public VideoDecoderSurfaceView(Context context) {
    this(context, /* attrs= */ null);
  }
+  /**
+   * Creates VideoDecoderSurfaceView.
+   *
+   * @param context A {@link Context}.
+   * @param attrs Custom attributes.
+   */
  public VideoDecoderSurfaceView(Context context, @Nullable AttributeSet attrs) {
    super(context, attrs);
-    renderer = new VideoDecoderRenderer();
+    renderer = new VideoDecoderRenderer(this);
    setPreserveEGLContextOnPause(true);
    setEGLContextClientVersion(2);
    setRenderer(renderer);
    setRenderMode(GLSurfaceView.RENDERMODE_WHEN_DIRTY);
  }
-  @Override
+  /**
-  public void setOutputBuffer(VideoDecoderOutputBuffer outputBuffer) {
+   * Returns the output buffer renderer used.
-    renderer.setFrame(outputBuffer);
+   *
-    requestRender();
+   * @return {@link VideoDecoderOutputBuffer}.
+   */
+  public VideoDecoderOutputBufferRenderer getOutputBufferRenderer() {
+    return renderer;
  }
 }
--- a/library/ui/src/main/java/com/google/android/exoplayer2/ui/PlayerView.java
+++ b/library/ui/src/main/java/com/google/android/exoplayer2/ui/PlayerView.java
@@ -64,6 +64,7 @@ import com.google.android.exoplayer2.util.Assertions;
 import com.google.android.exoplayer2.util.ErrorMessageProvider;
 import com.google.android.exoplayer2.util.RepeatModeUtil;
 import com.google.android.exoplayer2.util.Util;
+import com.google.android.exoplayer2.video.VideoDecoderSurfaceView;
 import com.google.android.exoplayer2.video.VideoListener;
 import java.lang.annotation.Documented;
 import java.lang.annotation.Retention;
@@ -276,6 +277,7 @@ public class PlayerView extends FrameLayout implements AdsLoader.AdViewProvider 
  private static final int SURFACE_TYPE_SURFACE_VIEW = 1;
  private static final int SURFACE_TYPE_TEXTURE_VIEW = 2;
  private static final int SURFACE_TYPE_MONO360_VIEW = 3;
+  private static final int SURFACE_TYPE_VIDEO_GL_SURFACE_VIEW = 4;
  // LINT.ThenChange(../../../../../../res/values/attrs.xml)
  @Nullable private final AspectRatioFrameLayout contentFrame;
@@ -411,6 +413,9 @@ public class PlayerView extends FrameLayout implements AdsLoader.AdViewProvider 
          sphericalSurfaceView.setSingleTapListener(componentListener);
          surfaceView = sphericalSurfaceView;
          break;
+        case SURFACE_TYPE_VIDEO_GL_SURFACE_VIEW:
+          surfaceView = new VideoDecoderSurfaceView(context);
+          break;
        default:
          surfaceView = new SurfaceView(context);
          break;
@@ -539,6 +544,8 @@ public class PlayerView extends FrameLayout implements AdsLoader.AdViewProvider 
          oldVideoComponent.clearVideoTextureView((TextureView) surfaceView);
        } else if (surfaceView instanceof SphericalSurfaceView) {
          ((SphericalSurfaceView) surfaceView).setVideoComponent(null);
+        } else if (surfaceView instanceof VideoDecoderSurfaceView) {
+          oldVideoComponent.setOutputBufferRenderer(null);
        } else if (surfaceView instanceof SurfaceView) {
          oldVideoComponent.clearVideoSurfaceView((SurfaceView) surfaceView);
        }
@@ -565,6 +572,9 @@ public class PlayerView extends FrameLayout implements AdsLoader.AdViewProvider 
          newVideoComponent.setVideoTextureView((TextureView) surfaceView);
        } else if (surfaceView instanceof SphericalSurfaceView) {
          ((SphericalSurfaceView) surfaceView).setVideoComponent(newVideoComponent);
+        } else if (surfaceView instanceof VideoDecoderSurfaceView) {
+          newVideoComponent.setOutputBufferRenderer(
+              ((VideoDecoderSurfaceView) surfaceView).getOutputBufferRenderer());
        } else if (surfaceView instanceof SurfaceView) {
          newVideoComponent.setVideoSurfaceView((SurfaceView) surfaceView);
        }
@@ -736,8 +746,8 @@ public class PlayerView extends FrameLayout implements AdsLoader.AdViewProvider 
   * buffering spinner is not displayed by default.
   *
   * @param showBuffering The mode that defines when the buffering spinner is displayed. One of
-   *     {@link #SHOW_BUFFERING_NEVER}, {@link #SHOW_BUFFERING_WHEN_PLAYING} and
+   *     {@link #SHOW_BUFFERING_NEVER}, {@link #SHOW_BUFFERING_WHEN_PLAYING} and {@link
-   *     {@link #SHOW_BUFFERING_ALWAYS}.
+   *     #SHOW_BUFFERING_ALWAYS}.
   */
  public void setShowBuffering(@ShowBuffering int showBuffering) {
    if (this.showBuffering != showBuffering) {

--- a/library/ui/src/main/res/values/attrs.xml
+++ b/library/ui/src/main/res/values/attrs.xml
@@ -30,6 +30,7 @@
    <enum name="surface_view" value="1"/>
    <enum name="texture_view" value="2"/>
    <enum name="spherical_view" value="3"/>
+    <enum name="video_decoder_surface_view" value="4"/>
  </attr>
  <!-- Must be kept in sync with RepeatModeUtil -->