Commit 2b4642f6 by hschlueter Committed by Marc Baechinger

Skip SSIM calculation on Nexus 5 API 21.

There is a problem with the ImageReader formats used by the
SSIM helper that only occurs for Nexus 5 API 21, so as a workaround
we can skip the SSIM calculation on Nexus 5 API 21.

This skips just the SSIM calculation (by setting the value to
1.0 instead and logging). The tests still run when SSIM is skipped
so that we can detect other failures.

PiperOrigin-RevId: 450903183
parent 52ad75c0
...@@ -37,6 +37,8 @@ import org.json.JSONObject; ...@@ -37,6 +37,8 @@ import org.json.JSONObject;
/** Utilities for instrumentation tests. */ /** Utilities for instrumentation tests. */
public final class AndroidTestUtil { public final class AndroidTestUtil {
private static final String TAG = "AndroidTestUtil";
// TODO(b/228865104): Add device capability based test skipping. // TODO(b/228865104): Add device capability based test skipping.
public static final String MP4_ASSET_URI_STRING = "asset:///media/mp4/sample.mp4"; public static final String MP4_ASSET_URI_STRING = "asset:///media/mp4/sample.mp4";
public static final Format MP4_ASSET_FORMAT = public static final Format MP4_ASSET_FORMAT =
...@@ -136,7 +138,7 @@ public final class AndroidTestUtil { ...@@ -136,7 +138,7 @@ public final class AndroidTestUtil {
*/ */
public static void recordTestSkipped(Context context, String testId, String reason) public static void recordTestSkipped(Context context, String testId, String reason)
throws JSONException, IOException { throws JSONException, IOException {
Log.i(testId, reason); Log.i(TAG, testId + ": " + reason);
JSONObject testJson = new JSONObject(); JSONObject testJson = new JSONObject();
testJson.put("skipReason", reason); testJson.put("skipReason", reason);
...@@ -216,7 +218,7 @@ public final class AndroidTestUtil { ...@@ -216,7 +218,7 @@ public final class AndroidTestUtil {
// Log contents as well as writing to file, for easier visibility on individual device testing. // Log contents as well as writing to file, for easier visibility on individual device testing.
for (String line : Util.split(analysisContents, "\n")) { for (String line : Util.split(analysisContents, "\n")) {
Log.i(testId, line); Log.i(TAG, testId + ": " + line);
} }
File analysisFile = createExternalCacheFile(context, /* fileName= */ testId + "-result.txt"); File analysisFile = createExternalCacheFile(context, /* fileName= */ testId + "-result.txt");
......
...@@ -27,6 +27,7 @@ import com.google.android.exoplayer2.Format; ...@@ -27,6 +27,7 @@ import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.MediaItem; import com.google.android.exoplayer2.MediaItem;
import com.google.android.exoplayer2.util.Log; import com.google.android.exoplayer2.util.Log;
import com.google.android.exoplayer2.util.SystemClock; import com.google.android.exoplayer2.util.SystemClock;
import com.google.android.exoplayer2.util.Util;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.List; import java.util.List;
...@@ -40,7 +41,7 @@ import org.json.JSONObject; ...@@ -40,7 +41,7 @@ import org.json.JSONObject;
/** An android instrumentation test runner for {@link Transformer}. */ /** An android instrumentation test runner for {@link Transformer}. */
public class TransformerAndroidTestRunner { public class TransformerAndroidTestRunner {
private static final String TAG_PREFIX = "TransformerAndroidTest_"; private static final String TAG = "TransformerAndroidTest";
/** The default transformation timeout value. */ /** The default transformation timeout value. */
public static final int DEFAULT_TIMEOUT_SECONDS = 120; public static final int DEFAULT_TIMEOUT_SECONDS = 120;
...@@ -299,7 +300,7 @@ public class TransformerAndroidTestRunner { ...@@ -299,7 +300,7 @@ public class TransformerAndroidTestRunner {
if (calculateSsim) { if (calculateSsim) {
double ssim = double ssim =
SsimHelper.calculate( SsimHelper.calculate(
context, /* expectedVideoPath= */ uriString, outputVideoFile.getPath()); context, /* referenceVideoPath= */ uriString, outputVideoFile.getPath());
resultBuilder.setSsim(ssim); resultBuilder.setSsim(ssim);
} }
} catch (InterruptedException interruptedException) { } catch (InterruptedException interruptedException) {
...@@ -308,15 +309,19 @@ public class TransformerAndroidTestRunner { ...@@ -308,15 +309,19 @@ public class TransformerAndroidTestRunner {
// TransformationTestResult. // TransformationTestResult.
throw interruptedException; throw interruptedException;
} catch (Throwable analysisFailure) { } catch (Throwable analysisFailure) {
// Catch all (checked and unchecked) failures throw by the SsimHelper and process them as if (Util.SDK_INT == 21 && "Nexus 5".equals(Util.MODEL)) { // b/233584640
// part of the TransformationTestResult. Log.i(TAG, testId + ": Skipping SSIM calculation due to known device-specific issue");
Exception analysisException = } else {
analysisFailure instanceof Exception // Catch all (checked and unchecked) failures throw by the SsimHelper and process them as
? (Exception) analysisFailure // part of the TransformationTestResult.
: new IllegalStateException(analysisFailure); Exception analysisException =
analysisFailure instanceof Exception
resultBuilder.setAnalysisException(analysisException); ? (Exception) analysisFailure
Log.e(TAG_PREFIX + testId, "SSIM calculation failed.", analysisException); : new IllegalStateException(analysisFailure);
resultBuilder.setAnalysisException(analysisException);
Log.e(TAG, testId + ": SSIM calculation failed.", analysisException);
}
} }
return resultBuilder.build(); return resultBuilder.build();
} }
......
...@@ -60,7 +60,9 @@ public final class TranscodeQualityTest { ...@@ -60,7 +60,9 @@ public final class TranscodeQualityTest {
.build() .build()
.run(testId, AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING); .run(testId, AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING);
assertThat(result.ssim).isGreaterThan(0.90); if (result.ssim != TransformationTestResult.SSIM_UNSET) {
assertThat(result.ssim).isGreaterThan(0.90);
}
} }
@Test @Test
...@@ -92,7 +94,9 @@ public final class TranscodeQualityTest { ...@@ -92,7 +94,9 @@ public final class TranscodeQualityTest {
.build() .build()
.run(testId, AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING); .run(testId, AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING);
assertThat(result.ssim).isGreaterThan(0.90); if (result.ssim != TransformationTestResult.SSIM_UNSET) {
assertThat(result.ssim).isGreaterThan(0.90);
}
} }
@Test @Test
...@@ -119,6 +123,8 @@ public final class TranscodeQualityTest { ...@@ -119,6 +123,8 @@ public final class TranscodeQualityTest {
testId, testId,
AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING); AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING);
assertThat(result.ssim).isGreaterThan(0.90); if (result.ssim != TransformationTestResult.SSIM_UNSET) {
assertThat(result.ssim).isGreaterThan(0.90);
}
} }
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment