test: Fix metrics-related timeouts in instrumentation tests and stabilize 16 KB emulator job

.github/workflows/test.yml

@@ -591,9 +591,13 @@ jobs:
       - name: 16 KB runtime qualification
         uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # v2.38.0
         with:
-          api-level: 35
+          # API 35 was the first 16 KB image; its ART runtime crashed system_server mid-run
+          # (NPE in ReferenceQueueDaemon), so qualify on the newer API 36 image instead.
+          api-level: 36
           arch: x86_64
           target: google_apis_ps16k
+          # Use all 4 vCPUs of the standard GitHub Linux runner instead of the default 2.
+          cores: 4
           # The ps16k google_apis image thrashes under the default AVD RAM: 16 KB pages inflate
           # every process, and system-wide mem-pressure kills reaped the test service process
           # (alongside the launcher and GMS) seconds after boot.

backtrace-library/src/androidTest/java/backtraceio/library/BacktraceClientMetricsTest.java

@@ -65,6 +65,11 @@ public void setUp() {
                 new BacktraceDatabase(context, context.getFilesDir().getAbsolutePath());
 
         backtraceClient = new BacktraceClient(context, credentials, database);
+
+        // metrics.enable() sends startup events right away; route them to a mock so no test reaches
+        // the real events endpoint. Tests replace these with their own mocks where they assert on them.
+        backtraceClient.metrics.setUniqueEventsRequestHandler(new MockRequestHandler());
+        backtraceClient.metrics.setSummedEventsRequestHandler(new MockRequestHandler());
     }
 
     public class MockRequestHandler implements EventsRequestHandler {
@@ -109,7 +114,7 @@ public void try3TimesOn503() {
 
         final int timeBetweenRetriesMillis = 1;
 
-        // Enable metrics first - this will send startup events with default handlers
+        // Enable metrics first - this will send startup events to the setUp() mock handlers
         backtraceClient.metrics.enable(
                 new BacktraceMetricsSettings(credentials, defaultBaseUrl, 0, timeBetweenRetriesMillis));
 

backtrace-library/src/androidTest/java/backtraceio/library/BacktraceClientSummedEventTest.java

@@ -51,6 +51,11 @@ public void setUp() {
                 new BacktraceDatabase(context, context.getFilesDir().getAbsolutePath());
 
         backtraceClient = new BacktraceClient(context, credentials, database);
+
+        // metrics.enable() sends startup events right away; route them to a mock so no test reaches
+        // the real events endpoint. Tests replace these with their own mocks where they assert on them.
+        backtraceClient.metrics.setUniqueEventsRequestHandler(new MockRequestHandler());
+        backtraceClient.metrics.setSummedEventsRequestHandler(new MockRequestHandler());
     }
 
     public class MockRequestHandler implements EventsRequestHandler {

backtrace-library/src/androidTest/java/backtraceio/library/BacktraceClientUniqueEventTest.java

@@ -69,6 +69,11 @@ public void setUp() {
                 new BacktraceDatabase(context, context.getFilesDir().getAbsolutePath());
 
         backtraceClient = new BacktraceClient(context, credentials, database);
+
+        // metrics.enable() sends startup events right away; route them to a mock so no test reaches
+        // the real events endpoint. Tests replace these with their own mocks where they assert on them.
+        backtraceClient.metrics.setUniqueEventsRequestHandler(new MockRequestHandler());
+        backtraceClient.metrics.setSummedEventsRequestHandler(new MockRequestHandler());
     }
 
     public class MockRequestHandler implements EventsRequestHandler {

example-app/src/androidTest/java/backtraceio/backtraceio/ExampleInstrumentedTest.java

@@ -6,14 +6,19 @@
 import static org.junit.Assert.assertEquals;
 
 import android.content.Context;
+import android.content.Intent;
+import android.os.SystemClock;
 import androidx.lifecycle.Lifecycle;
+import androidx.test.core.app.ApplicationProvider;
 import androidx.test.ext.junit.rules.ActivityScenarioRule;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
 import androidx.test.platform.app.InstrumentationRegistry;
 import backtraceio.coroner.response.CoronerResponse;
 import backtraceio.coroner.response.CoronerResponseProcessingException;
 import backtraceio.library.logger.BacktraceLogger;
 import backtraceio.library.logger.LogLevel;
+import backtraceio.library.models.BacktraceResult;
+import backtraceio.library.models.types.BacktraceResultStatus;
 import java.util.Arrays;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
@@ -32,12 +37,18 @@
 @RunWith(AndroidJUnit4.class)
 public class ExampleInstrumentedTest extends InstrumentedTest {
     private final int THREAD_SLEEP_TIME_MS = 2000;
+    private static final long SUBMISSION_TIMEOUT_MS = 60_000;
+    private static final long INGESTION_TIMEOUT_MS = 60_000;
 
     @Rule
-    public ActivityScenarioRule<MainActivity> mActivityRule = new ActivityScenarioRule<>(MainActivity.class);
+    public ActivityScenarioRule<MainActivity> mActivityRule =
+            new ActivityScenarioRule<>(new Intent(ApplicationProvider.getApplicationContext(), MainActivity.class)
+                    // Metrics share the sender thread with reports, so an unreachable events endpoint
+                    // would delay the uploads these tests wait for. None of these tests assert on metrics.
+                    .putExtra(MainActivity.EXTRA_DISABLE_METRICS, true));
 
     @Before
-    public void enableMetricsAndBreadcrumbs() {
+    public void enableBreadcrumbs() {
         BacktraceLogger.setLevel(LogLevel.DEBUG);
         // Re-assert RESUMED: some old real devices background the activity right after launch.
         mActivityRule.getScenario().moveToState(Lifecycle.State.RESUMED);
@@ -54,30 +65,27 @@ public void useAppContext() {
     @Test
     public void handledException() throws TimeoutException, CoronerResponseProcessingException, InterruptedException {
         // GIVEN
-        final String[] rxId = new String[1];
+        final BacktraceResult[] result = new BacktraceResult[1];
         final Waiter waiter = new Waiter();
         mActivityRule
                 .getScenario()
                 .onActivity(activity -> activity.setOnServerResponseEventListener(backtraceResult -> {
-                    rxId[0] = backtraceResult.rxId;
+                    result[0] = backtraceResult;
                     waiter.resume();
                 }));
 
         // WHEN
         onView(withId(R.id.handledException)).perform(click()); // UI action
-        waiter.await(5, TimeUnit.SECONDS, 1);
-        Thread.sleep(THREAD_SLEEP_TIME_MS);
+        // A live round trip from a Sauce device to the submission endpoint, queued behind any other
+        // upload on the single sender thread, routinely exceeds a few seconds.
+        waiter.await(SUBMISSION_TIMEOUT_MS, TimeUnit.MILLISECONDS, 1);
+        Assert.assertNotNull("No submission result", result[0]);
+        Assert.assertEquals("Submission failed: " + result[0].message, BacktraceResultStatus.Ok, result[0].status);
+        String rxId = result[0].rxId;
 
         // THEN
-        CoronerResponse response = null;
+        CoronerResponse response = awaitIngestedReport(rxId);
 
-        try {
-            response = this.getCoronerClient().rxIdFilter(rxId[0], Arrays.asList("error.message"));
-        } catch (Exception ex) {
-            Assert.fail(ex.getMessage());
-        }
-
-        Assert.assertNotNull(response);
         Assert.assertEquals(1, response.getResultsNumber());
 
         String resultErrorMsg = response.getAttribute(0, "error.message", String.class);
@@ -87,6 +95,30 @@ public void handledException() throws TimeoutException, CoronerResponseProcessin
         Assert.assertEquals("java.lang.IndexOutOfBoundsException", resultClassifier);
     }
 
+    /**
+     * Polls Coroner until the report with the given rxId is queryable. Ingestion is asynchronous on
+     * the server, so a single query right after the upload can miss a report that was accepted.
+     */
+    private CoronerResponse awaitIngestedReport(String rxId) throws InterruptedException {
+        long deadline = SystemClock.elapsedRealtime() + INGESTION_TIMEOUT_MS;
+        Exception lastPollFailure = null;
+        while (SystemClock.elapsedRealtime() < deadline) {
+            try {
+                CoronerResponse response = this.getCoronerClient().rxIdFilter(rxId, Arrays.asList("error.message"));
+                lastPollFailure = null;
+                if (response != null && response.getResultsNumber() > 0) {
+                    return response;
+                }
+            } catch (Exception ex) {
+                lastPollFailure = ex;
+            }
+            Thread.sleep(THREAD_SLEEP_TIME_MS);
+        }
+        Assert.fail("Report " + rxId + " not ingested within " + INGESTION_TIMEOUT_MS + " ms"
+                + (lastPollFailure == null ? "" : "; last poll failure: " + lastPollFailure.getMessage()));
+        return null;
+    }
+
     @Test
     public void dumpWithoutCrash() throws CoronerResponseProcessingException, InterruptedException {
         // GIVEN

example-app/src/main/java/backtraceio/backtraceio/MainActivity.java

@@ -32,6 +32,11 @@
 import java.util.Map;
 
 public class MainActivity extends AppCompatActivity {
+    /**
+     * Intent extra that skips enabling Backtrace metrics. Instrumentation tests set it so startup
+     * metrics requests don't share the sender thread with the reports under test.
+     */
+    public static final String EXTRA_DISABLE_METRICS = "backtraceio.backtraceio.DISABLE_METRICS";
 
     private BacktraceClient backtraceClient;
     private OnServerResponseEventListener listener;
@@ -51,7 +56,8 @@ protected void onCreate(Bundle savedInstanceState) {
         super.onCreate(savedInstanceState);
         setContentView(R.layout.activity_main);
 
-        backtraceClient = initializeBacktrace(BuildConfig.BACKTRACE_SUBMISSION_URL);
+        boolean enableMetrics = !getIntent().getBooleanExtra(EXTRA_DISABLE_METRICS, false);
+        backtraceClient = initializeBacktrace(BuildConfig.BACKTRACE_SUBMISSION_URL, enableMetrics);
 
         symlinkAndWriteFile();
     }
@@ -72,7 +78,7 @@ private void symlinkAndWriteFile() {
         writeMyCustomFile(fileNameDateString);
     }
 
-    private BacktraceClient initializeBacktrace(final String submissionUrl) {
+    private BacktraceClient initializeBacktrace(final String submissionUrl, final boolean enableMetrics) {
         BacktraceCredentials credentials = new BacktraceCredentials(submissionUrl);
 
         Context context = getApplicationContext();
@@ -102,7 +108,9 @@ private BacktraceClient initializeBacktrace(final String submissionUrl) {
 
         BacktraceExceptionHandler.enable(backtraceClient);
 
-        backtraceClient.metrics.enable();
+        if (enableMetrics) {
+            backtraceClient.metrics.enable();
+        }
 
         // Enable handling of native crashes
         database.setupNativeIntegration(backtraceClient, credentials, true);