vp9_extension: Fix potential integer overflows

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=138652561

extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/VpxDecoder.java

@@ -122,8 +122,11 @@ import java.nio.ByteBuffer;
     }
     }
 
 
     outputBuffer.init(inputBuffer.timeUs, outputMode);
     outputBuffer.init(inputBuffer.timeUs, outputMode);
-    if (vpxGetFrame(vpxDecContext, outputBuffer) != 0) {
+    int getFrameResult = vpxGetFrame(vpxDecContext, outputBuffer);
+    if (getFrameResult == 1) {
       outputBuffer.addFlag(C.BUFFER_FLAG_DECODE_ONLY);
       outputBuffer.addFlag(C.BUFFER_FLAG_DECODE_ONLY);
+    } else if (getFrameResult == -1) {
+      return new VpxDecoderException("Buffer initialization failed.");
     }
     }
     return null;
     return null;
   }
   }

extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/VpxOutputBuffer.java

@@ -66,28 +66,39 @@ import java.nio.ByteBuffer;
 
 
   /**
   /**
    * Resizes the buffer based on the given dimensions. Called via JNI after decoding completes.
    * Resizes the buffer based on the given dimensions. Called via JNI after decoding completes.
+   * @return Whether the buffer was resized successfully.
    */
    */
-  public void initForRgbFrame(int width, int height) {
+  public boolean initForRgbFrame(int width, int height) {
     this.width = width;
     this.width = width;
     this.height = height;
     this.height = height;
     this.yuvPlanes = null;
     this.yuvPlanes = null;
-
+    if (!isSafeToMultiply(width, height) || !isSafeToMultiply(width * height, 2)) {
+      return false;
+    }
     int minimumRgbSize = width * height * 2;
     int minimumRgbSize = width * height * 2;
     initData(minimumRgbSize);
     initData(minimumRgbSize);
+    return true;
   }
   }
 
 
   /**
   /**
    * Resizes the buffer based on the given stride. Called via JNI after decoding completes.
    * Resizes the buffer based on the given stride. Called via JNI after decoding completes.
+   * @return Whether the buffer was resized successfully.
    */
    */
-  public void initForYuvFrame(int width, int height, int yStride, int uvStride,
+  public boolean initForYuvFrame(int width, int height, int yStride, int uvStride,
       int colorspace) {
       int colorspace) {
     this.width = width;
     this.width = width;
     this.height = height;
     this.height = height;
     this.colorspace = colorspace;
     this.colorspace = colorspace;
-
+    int uvHeight = (int) (((long) height + 1) / 2);
+    if (!isSafeToMultiply(yStride, height) || !isSafeToMultiply(uvStride, uvHeight)) {
+      return false;
+    }
     int yLength = yStride * height;
     int yLength = yStride * height;
-    int uvLength = uvStride * ((height + 1) / 2);
+    int uvLength = uvStride * uvHeight;
     int minimumYuvSize = yLength + (uvLength * 2);
     int minimumYuvSize = yLength + (uvLength * 2);
+    if (!isSafeToMultiply(uvLength, 2) || minimumYuvSize < yLength) {
+      return false;
+    }
     initData(minimumYuvSize);
     initData(minimumYuvSize);
 
 
     if (yuvPlanes == null) {
     if (yuvPlanes == null) {
@@ -108,6 +119,7 @@ import java.nio.ByteBuffer;
     yuvStrides[0] = yStride;
     yuvStrides[0] = yStride;
     yuvStrides[1] = uvStride;
     yuvStrides[1] = uvStride;
     yuvStrides[2] = uvStride;
     yuvStrides[2] = uvStride;
+    return true;
   }
   }
 
 
   private void initData(int size) {
   private void initData(int size) {
@@ -119,4 +131,12 @@ import java.nio.ByteBuffer;
     }
     }
   }
   }
 
 
+  /**
+   * Ensures that the result of multiplying individual numbers can fit into the size limit of an
+   * integer.
+   */
+  private boolean isSafeToMultiply(int a, int b) {
+    return a >= 0 && b >= 0 && !(b > 0 && a >= Integer.MAX_VALUE / b);
+  }
+
 }
 }

extensions/vp9/src/main/jni/vpx_jni.cc

@@ -83,9 +83,9 @@ DECODER_FUNC(jlong, vpxInit) {
   const jclass outputBufferClass = env->FindClass(
   const jclass outputBufferClass = env->FindClass(
       "com/google/android/exoplayer2/ext/vp9/VpxOutputBuffer");
       "com/google/android/exoplayer2/ext/vp9/VpxOutputBuffer");
   initForYuvFrame = env->GetMethodID(outputBufferClass, "initForYuvFrame",
   initForYuvFrame = env->GetMethodID(outputBufferClass, "initForYuvFrame",
-                                     "(IIIII)V");
+                                     "(IIIII)Z");
   initForRgbFrame = env->GetMethodID(outputBufferClass, "initForRgbFrame",
   initForRgbFrame = env->GetMethodID(outputBufferClass, "initForRgbFrame",
-                                     "(II)V");
+                                     "(II)Z");
   dataField = env->GetFieldID(outputBufferClass, "data",
   dataField = env->GetFieldID(outputBufferClass, "data",
                               "Ljava/nio/ByteBuffer;");
                               "Ljava/nio/ByteBuffer;");
   outputModeField = env->GetFieldID(outputBufferClass, "mode", "I");
   outputModeField = env->GetFieldID(outputBufferClass, "mode", "I");
@@ -141,7 +141,11 @@ DECODER_FUNC(jint, vpxGetFrame, jlong jContext, jobject jOutputBuffer) {
   int outputMode = env->GetIntField(jOutputBuffer, outputModeField);
   int outputMode = env->GetIntField(jOutputBuffer, outputModeField);
   if (outputMode == kOutputModeRgb) {
   if (outputMode == kOutputModeRgb) {
     // resize buffer if required.
     // resize buffer if required.
-    env->CallVoidMethod(jOutputBuffer, initForRgbFrame, img->d_w, img->d_h);
+    jboolean initResult = env->CallBooleanMethod(jOutputBuffer, initForRgbFrame,
+                                                 img->d_w, img->d_h);
+    if (initResult == JNI_FALSE) {
+      return -1;
+    }
 
 
     // get pointer to the data buffer.
     // get pointer to the data buffer.
     const jobject dataObject = env->GetObjectField(jOutputBuffer, dataField);
     const jobject dataObject = env->GetObjectField(jOutputBuffer, dataField);
@@ -170,9 +174,12 @@ DECODER_FUNC(jint, vpxGetFrame, jlong jContext, jobject jOutputBuffer) {
     }
     }
 
 
     // resize buffer if required.
     // resize buffer if required.
-    env->CallVoidMethod(jOutputBuffer, initForYuvFrame, img->d_w, img->d_h,
-                        img->stride[VPX_PLANE_Y], img->stride[VPX_PLANE_U],
-                        colorspace);
+    jboolean initResult = env->CallBooleanMethod(
+        jOutputBuffer, initForYuvFrame, img->d_w, img->d_h,
+        img->stride[VPX_PLANE_Y], img->stride[VPX_PLANE_U], colorspace);
+    if (initResult == JNI_FALSE) {
+      return -1;
+    }
 
 
     // get pointer to the data buffer.
     // get pointer to the data buffer.
     const jobject dataObject = env->GetObjectField(jOutputBuffer, dataField);
     const jobject dataObject = env->GetObjectField(jOutputBuffer, dataField);