Fix video decoder GL rendering

Calculate stride and width ratio per each plane.

PiperOrigin-RevId: 272916423

library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderRenderer.java

@@ -17,6 +17,7 @@ package com.google.android.exoplayer2.video;
 
 
 import android.opengl.GLES20;
 import android.opengl.GLES20;
 import android.opengl.GLSurfaceView;
 import android.opengl.GLSurfaceView;
+import com.google.android.exoplayer2.util.Assertions;
 import com.google.android.exoplayer2.util.GlUtil;
 import com.google.android.exoplayer2.util.GlUtil;
 import java.nio.FloatBuffer;
 import java.nio.FloatBuffer;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.atomic.AtomicReference;
@@ -48,50 +49,64 @@ import javax.microedition.khronos.opengles.GL10;
   };
   };
 
 
   private static final String VERTEX_SHADER =
   private static final String VERTEX_SHADER =
-      "varying vec2 interp_tc;\n"
-      + "attribute vec4 in_pos;\n"
-      + "attribute vec2 in_tc;\n"
-      + "void main() {\n"
-      + "  gl_Position = in_pos;\n"
-      + "  interp_tc = in_tc;\n"
-      + "}\n";
+      "varying vec2 interp_tc_y;\n"
+          + "varying vec2 interp_tc_u;\n"
+          + "varying vec2 interp_tc_v;\n"
+          + "attribute vec4 in_pos;\n"
+          + "attribute vec2 in_tc_y;\n"
+          + "attribute vec2 in_tc_u;\n"
+          + "attribute vec2 in_tc_v;\n"
+          + "void main() {\n"
+          + "  gl_Position = in_pos;\n"
+          + "  interp_tc_y = in_tc_y;\n"
+          + "  interp_tc_u = in_tc_u;\n"
+          + "  interp_tc_v = in_tc_v;\n"
+          + "}\n";
   private static final String[] TEXTURE_UNIFORMS = {"y_tex", "u_tex", "v_tex"};
   private static final String[] TEXTURE_UNIFORMS = {"y_tex", "u_tex", "v_tex"};
   private static final String FRAGMENT_SHADER =
   private static final String FRAGMENT_SHADER =
       "precision mediump float;\n"
       "precision mediump float;\n"
-      + "varying vec2 interp_tc;\n"
-      + "uniform sampler2D y_tex;\n"
-      + "uniform sampler2D u_tex;\n"
-      + "uniform sampler2D v_tex;\n"
-      + "uniform mat3 mColorConversion;\n"
-      + "void main() {\n"
-      + "  vec3 yuv;\n"
-      + "  yuv.x = texture2D(y_tex, interp_tc).r - 0.0625;\n"
-      + "  yuv.y = texture2D(u_tex, interp_tc).r - 0.5;\n"
-      + "  yuv.z = texture2D(v_tex, interp_tc).r - 0.5;\n"
-      + "  gl_FragColor = vec4(mColorConversion * yuv, 1.0);\n"
-      + "}\n";
+          + "varying vec2 interp_tc_y;\n"
+          + "varying vec2 interp_tc_u;\n"
+          + "varying vec2 interp_tc_v;\n"
+          + "uniform sampler2D y_tex;\n"
+          + "uniform sampler2D u_tex;\n"
+          + "uniform sampler2D v_tex;\n"
+          + "uniform mat3 mColorConversion;\n"
+          + "void main() {\n"
+          + "  vec3 yuv;\n"
+          + "  yuv.x = texture2D(y_tex, interp_tc_y).r - 0.0625;\n"
+          + "  yuv.y = texture2D(u_tex, interp_tc_u).r - 0.5;\n"
+          + "  yuv.z = texture2D(v_tex, interp_tc_v).r - 0.5;\n"
+          + "  gl_FragColor = vec4(mColorConversion * yuv, 1.0);\n"
+          + "}\n";
 
 
   private static final FloatBuffer TEXTURE_VERTICES =
   private static final FloatBuffer TEXTURE_VERTICES =
       GlUtil.createBuffer(new float[] {-1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f});
       GlUtil.createBuffer(new float[] {-1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f});
   private final int[] yuvTextures = new int[3];
   private final int[] yuvTextures = new int[3];
   private final AtomicReference<VideoDecoderOutputBuffer> pendingOutputBufferReference;
   private final AtomicReference<VideoDecoderOutputBuffer> pendingOutputBufferReference;
 
 
-  // Kept in a field rather than a local variable so that it doesn't get garbage collected before
+  // Kept in field rather than a local variable in order not to get garbage collected before
   // glDrawArrays uses it.
   // glDrawArrays uses it.
   @SuppressWarnings("FieldCanBeLocal")
   @SuppressWarnings("FieldCanBeLocal")
-  private FloatBuffer textureCoords;
+  private FloatBuffer[] textureCoords;
+
   private int program;
   private int program;
-  private int texLocation;
+  private int[] texLocations;
   private int colorMatrixLocation;
   private int colorMatrixLocation;
-  private int previousWidth;
-  private int previousStride;
+  private int[] previousWidths;
+  private int[] previousStrides;
 
 
   private VideoDecoderOutputBuffer renderedOutputBuffer; // Accessed only from the GL thread.
   private VideoDecoderOutputBuffer renderedOutputBuffer; // Accessed only from the GL thread.
 
 
   public VideoDecoderRenderer() {
   public VideoDecoderRenderer() {
-    previousWidth = -1;
-    previousStride = -1;
     pendingOutputBufferReference = new AtomicReference<>();
     pendingOutputBufferReference = new AtomicReference<>();
+    textureCoords = new FloatBuffer[3];
+    texLocations = new int[3];
+    previousWidths = new int[3];
+    previousStrides = new int[3];
+    for (int i = 0; i < 3; i++) {
+      previousWidths[i] = previousStrides[i] = -1;
+    }
   }
   }
 
 
   /**
   /**
@@ -115,10 +130,13 @@ import javax.microedition.khronos.opengles.GL10;
     GLES20.glUseProgram(program);
     GLES20.glUseProgram(program);
     int posLocation = GLES20.glGetAttribLocation(program, "in_pos");
     int posLocation = GLES20.glGetAttribLocation(program, "in_pos");
     GLES20.glEnableVertexAttribArray(posLocation);
     GLES20.glEnableVertexAttribArray(posLocation);
-    GLES20.glVertexAttribPointer(
-        posLocation, 2, GLES20.GL_FLOAT, false, 0, TEXTURE_VERTICES);
-    texLocation = GLES20.glGetAttribLocation(program, "in_tc");
-    GLES20.glEnableVertexAttribArray(texLocation);
+    GLES20.glVertexAttribPointer(posLocation, 2, GLES20.GL_FLOAT, false, 0, TEXTURE_VERTICES);
+    texLocations[0] = GLES20.glGetAttribLocation(program, "in_tc_y");
+    GLES20.glEnableVertexAttribArray(texLocations[0]);
+    texLocations[1] = GLES20.glGetAttribLocation(program, "in_tc_u");
+    GLES20.glEnableVertexAttribArray(texLocations[1]);
+    texLocations[2] = GLES20.glGetAttribLocation(program, "in_tc_v");
+    GLES20.glEnableVertexAttribArray(texLocations[2]);
     GlUtil.checkGlError();
     GlUtil.checkGlError();
     colorMatrixLocation = GLES20.glGetUniformLocation(program, "mColorConversion");
     colorMatrixLocation = GLES20.glGetUniformLocation(program, "mColorConversion");
     GlUtil.checkGlError();
     GlUtil.checkGlError();
@@ -165,22 +183,41 @@ import javax.microedition.khronos.opengles.GL10;
       GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i);
       GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i);
       GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]);
       GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]);
       GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1);
       GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1);
-      GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE,
-          outputBuffer.yuvStrides[i], h, 0, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE,
+      GLES20.glTexImage2D(
+          GLES20.GL_TEXTURE_2D,
+          0,
+          GLES20.GL_LUMINANCE,
+          outputBuffer.yuvStrides[i],
+          h,
+          0,
+          GLES20.GL_LUMINANCE,
+          GLES20.GL_UNSIGNED_BYTE,
           outputBuffer.yuvPlanes[i]);
           outputBuffer.yuvPlanes[i]);
     }
     }
-    // Set cropping of stride if either width or stride has changed.
-    if (previousWidth != outputBuffer.width || previousStride != outputBuffer.yuvStrides[0]) {
-      float crop = (float) outputBuffer.width / outputBuffer.yuvStrides[0];
-      // This buffer is consumed during each call to glDrawArrays. It needs to be a member variable
-      // rather than a local variable to ensure that it doesn't get garbage collected.
-      textureCoords =
-          GlUtil.createBuffer(new float[] {0.0f, 0.0f, 0.0f, 1.0f, crop, 0.0f, crop, 1.0f});
-      GLES20.glVertexAttribPointer(
-          texLocation, 2, GLES20.GL_FLOAT, false, 0, textureCoords);
-      previousWidth = outputBuffer.width;
-      previousStride = outputBuffer.yuvStrides[0];
+
+    int[] widths = new int[3];
+    widths[0] = outputBuffer.width;
+    // TODO: Handle streams where chroma channels are not stored at half width and height
+    // compared to luma channel. See [Internal: b/142097774].
+    // U and V planes are being stored at half width compared to Y.
+    widths[1] = widths[2] = (widths[0] + 1) / 2;
+    for (int i = 0; i < 3; i++) {
+      // Set cropping of stride if either width or stride has changed.
+      if (previousWidths[i] != widths[i] || previousStrides[i] != outputBuffer.yuvStrides[i]) {
+        Assertions.checkState(outputBuffer.yuvStrides[i] != 0);
+        float widthRatio = (float) widths[i] / outputBuffer.yuvStrides[i];
+        // These buffers are consumed during each call to glDrawArrays. They need to be member
+        // variables rather than local variables in order not to get garbage collected.
+        textureCoords[i] =
+            GlUtil.createBuffer(
+                new float[] {0.0f, 0.0f, 0.0f, 1.0f, widthRatio, 0.0f, widthRatio, 1.0f});
+        GLES20.glVertexAttribPointer(
+            texLocations[i], 2, GLES20.GL_FLOAT, false, 0, textureCoords[i]);
+        previousWidths[i] = widths[i];
+        previousStrides[i] = outputBuffer.yuvStrides[i];
+      }
     }
     }
+
     GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
     GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
     GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
     GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
     GlUtil.checkGlError();
     GlUtil.checkGlError();
@@ -188,18 +225,16 @@ import javax.microedition.khronos.opengles.GL10;
 
 
   private void setupTextures() {
   private void setupTextures() {
     GLES20.glGenTextures(3, yuvTextures, 0);
     GLES20.glGenTextures(3, yuvTextures, 0);
-    for (int i = 0; i < 3; i++)  {
+    for (int i = 0; i < 3; i++) {
       GLES20.glUniform1i(GLES20.glGetUniformLocation(program, TEXTURE_UNIFORMS[i]), i);
       GLES20.glUniform1i(GLES20.glGetUniformLocation(program, TEXTURE_UNIFORMS[i]), i);
       GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i);
       GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i);
       GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]);
       GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]);
-      GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
-          GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR);
-      GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
-          GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
-      GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
-          GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
-      GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
-          GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
+      GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR);
+      GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
+      GLES20.glTexParameterf(
+          GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
+      GLES20.glTexParameterf(
+          GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
     }
     }
     GlUtil.checkGlError();
     GlUtil.checkGlError();
   }
   }