Prototype Android emulator containers for runtime and library tests

docs/workflow/building/coreclr/android.md

@@ -176,6 +176,36 @@ To build and run the test on Android with CoreCLR, run the following command fro
 > [!NOTE]
 > Similarly to the `HelloAndroid` sample, the emulator needs to be up and running.
 
+### Testing in a container
+
+The `runtime-androidemulator` pipeline has an opt-in `runAndroidContainerTests`
+parameter for API 29 parity validation. It builds CoreCLR, the
+`JIT/Directed/Directed_r.csproj` merged test assembly and the library test
+archives once. Each payload is submitted through its normal Helix path to both
+`Ubuntu.2204.Amd64.Android.29.Open` and the KVM-enabled container queue using a
+pinned Android API 29 image. Library submission uses the normal inner-loop test
+scope, including networking, IO and globalization rather than only Numerics.
+On the container queue each work item starts its own container and emulator;
+the legacy queue retains its host-managed emulator. The default emulator and
+device jobs are unchanged.
+
+Helix job names identify the destination queue; library runs also use the
+`CoreCLR_Release_API29_Parity_Libraries` prefix/suffix. Compare individual test
+identities and outcomes, skipped tests, work-item failures and durations across
+the two queues. A green run alone does not establish parity, and test-result
+comparison is not automated by this pipeline. This lane does not yet cover the
+full CoreCLR runtime suite, NativeAOT or other Android API levels.
+
+The container image supplies the Android SDK, system image and AVD.
+`eng/testing/Android/start-emulator.sh` and `stop-emulator.sh` own the work-item
+emulator lifecycle. Both libraries and CoreCLR Helix submissions import
+`eng/testing/Android/helix.targets` to include these scripts for Android container
+queues; legacy queues and Windows device queues do not receive the hooks.
+
+The container host must provide `/dev/kvm` and read/write access for the container
+user. The scripts verify access without changing the host or device permissions.
+They do not enable root access or core-dump collection inside Android.
+
 ## Debugging the runtime and the sample app
 
 Managed debugging is currently not supported, but we can debug:

eng/pipelines/runtime-androidemulator.yml

@@ -5,10 +5,18 @@
 
 trigger: none
 
+parameters:
+  - name: runAndroidContainerTests
+    displayName: Run CoreCLR and library API 29 parity validation only
+    type: boolean
+    default: false
+
 variables:
   - template: /eng/pipelines/common/variables.yml
   - name: enableHelixJobMonitor
     value: true
+  - name: androidContainerHelixQueue
+    value: (Android.29.Amd64.Open)Ubuntu.2204.Amd64.Android.Multi.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-24.04-helix-android-29-amd64@sha256:193e6d0ce33f6d8b9c10170ef8a197683c0fa4fe75fe5f1e3199318e8fb84d03
 
 extends:
   template:  /eng/pipelines/common/templates/pipeline-with-resources.yml
@@ -23,8 +31,60 @@ extends:
             timeoutInMinutes: 540
             allowNoHelixJobs: true
 
-      - template: /eng/pipelines/extra-platforms/runtime-extra-platforms-androidemulator.yml
-        parameters:
-          isExtraPlatformsBuild: ${{ variables.isExtraPlatformsBuild }}
-          isAndroidEmulatorOnlyBuild: ${{ variables.isAndroidEmulatorOnlyBuild }}
-          useHelixMonitor: ${{ variables.enableHelixJobMonitor }}
+      - ${{ if parameters.runAndroidContainerTests }}:
+        - template: /eng/pipelines/common/xplat-setup.yml
+          parameters:
+            jobTemplate: /eng/pipelines/common/global-build-job.yml
+            osGroup: android
+            archType: x64
+            targetRid: android-x64
+            platform: android_x64
+            container: android
+            variables:
+              - name: priorityArg
+                value: ''
+              - name: runtimeFlavorArgs
+                value: ''
+              - name: runtimeVariantArg
+                value: ''
+              - name: librariesConfigurationArg
+                value: ''
+            jobParameters:
+              buildConfig: Release
+              runtimeFlavor: coreclr
+              nameSuffix: API29_Parity_CoreCLR
+              buildArgs: -s clr.runtime+clr.alljits+clr.corelib+clr.nativecorelib+clr.tools+clr.packages+libs+libs.tests+host+packs -c Release /p:ArchiveTests=true
+              timeoutInMinutes: 240
+              helixQueues:
+                - Ubuntu.2204.Amd64.Android.29.Open
+                - $(androidContainerHelixQueue)
+              postBuildSteps:
+                - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests.yml
+                  parameters:
+                    testBuildArgs: test JIT/Directed/Directed_r.csproj
+                - template: /eng/pipelines/common/templates/runtimes/send-to-helix-step.yml
+                  parameters:
+                    displayName: Run CoreCLR JIT tests on API 29 legacy and container queues
+                    creator: $(Build.DefinitionName)
+                    helixBuild: $(Build.BuildNumber)
+                    helixSource: ci/dotnet/runtime/$(Build.SourceBranch)
+                    helixType: test/functional/cli/
+                    publishTestResults: true
+                    useHelixMonitor: true
+                    scenarios:
+                      - normal
+                    timeoutPerTestInMinutes: 10
+                    timeoutPerTestCollectionInMinutes: 30
+                    helixProjectArguments: $(Build.SourcesDirectory)/src/tests/Common/helixpublishwitharcade.proj
+                - template: /eng/pipelines/libraries/helix.yml
+                  parameters:
+                    creator: $(Build.DefinitionName)
+                    testRunNamePrefixSuffix: CoreCLR_Release_API29_Parity_Libraries
+                    useHelixMonitor: true
+
+      - ${{ if not(parameters.runAndroidContainerTests) }}:
+        - template: /eng/pipelines/extra-platforms/runtime-extra-platforms-androidemulator.yml
+          parameters:
+            isExtraPlatformsBuild: ${{ variables.isExtraPlatformsBuild }}
+            isAndroidEmulatorOnlyBuild: ${{ variables.isAndroidEmulatorOnlyBuild }}
+            useHelixMonitor: ${{ variables.enableHelixJobMonitor }}

eng/testing/Android/helix.targets

@@ -0,0 +1,9 @@
+<Project>
+  <ItemGroup Condition="'$(TargetOS)' == 'android' and
+                        $(HelixTargetQueue.Contains('@')) and
+                        !$(HelixTargetQueue.ToLowerInvariant().Contains('windows'))">
+    <HelixCorrelationPayload Include="$(MSBuildThisFileDirectory)" Destination="android-emulator" />
+    <HelixPreCommand Include="bash &quot;$HELIX_CORRELATION_PAYLOAD/android-emulator/start-emulator.sh&quot;" />
+    <HelixPostCommand Include="bash &quot;$HELIX_CORRELATION_PAYLOAD/android-emulator/stop-emulator.sh&quot;" />
+  </ItemGroup>
+</Project>

eng/testing/Android/start-emulator.sh

@@ -0,0 +1,149 @@
+#!/usr/bin/env bash
+
+# Boots the Android emulator that is baked into the Helix Android container image and waits
+# until it can accept an APK install.
+#
+# On the historical Ubuntu.2204.Amd64.Android.<api> queues the emulator was started by a systemd
+# unit at machine provisioning time and shared by every work item. Container based queues have no
+# systemd, so each work item boots its own emulator through this script (wired up as a Helix
+# pre-command). A per-work-item emulator also removes the state bleed between work items that the
+# shared emulator suffered from.
+#
+# Emulator failures here are infrastructure failures, not test failures, so they request a Helix
+# retry rather than reporting a failed test run.
+
+set -uo pipefail
+
+avd_name="${ANDROID_AVD_NAME:-}"
+boot_timeout_seconds="${ANDROID_EMULATOR_BOOT_TIMEOUT:-300}"
+boot_attempts="${ANDROID_EMULATOR_BOOT_ATTEMPTS:-2}"
+android_sdk_root="${ANDROID_SDK_ROOT:-${ANDROID_HOME:-/usr/local/android-sdk}}"
+
+emulator_binary="$android_sdk_root/emulator/emulator"
+adb_binary="$android_sdk_root/platform-tools/adb"
+
+# Helix picks these up to retry the work item on a different machine instead of reporting failures.
+report_infrastructure_failure() {
+    echo "Android emulator infrastructure failure: $1"
+
+    if [ -n "${HELIX_WORKITEM_ROOT:-}" ]; then
+        echo "$1" > "$HELIX_WORKITEM_ROOT/.retry"
+        echo "$1" > "$HELIX_WORKITEM_ROOT/.reboot"
+    fi
+}
+
+die() {
+    report_infrastructure_failure "$1"
+    exit 1
+}
+
+if [ -x "$adb_binary" ]; then
+    "$adb_binary" start-server > /dev/null 2>&1
+fi
+
+# The legacy Ubuntu.2204.Amd64.Android.<api> queues boot a shared emulator from a systemd unit
+# before the Helix client starts, so on those queues there is already a device and nothing to do.
+# Detecting that here lets the very same work item payload run on both the legacy queues and the
+# container queues, so the two can be migrated independently.
+if [ -x "$adb_binary" ] && \
+   [ "$("$adb_binary" shell getprop sys.boot_completed 2>/dev/null | tr -d '\r\n')" = "1" ]; then
+    echo "A booted Android device is already attached; leaving emulator lifecycle to the host."
+    "$adb_binary" devices -l
+    exit 0
+fi
+
+if [ -z "$avd_name" ]; then
+    # The image bakes in exactly one AVD, so fall back to whatever it is called.
+    avd_name="$("$emulator_binary" -list-avds 2>/dev/null | head -n 1)"
+fi
+
+[ -n "$avd_name" ] || die "No AVD found. Set ANDROID_AVD_NAME or bake an AVD into the image."
+[ -x "$emulator_binary" ] || die "Emulator not found at '$emulator_binary'."
+[ -x "$adb_binary" ] || die "adb not found at '$adb_binary'."
+
+# The emulator needs hardware acceleration; without KVM it silently falls back to full instruction
+# emulation, which is slow enough to blow through every work item timeout.
+#
+# Providing the device, and access to it, is the infrastructure's job: the image declares that it
+# needs /dev/kvm and the Helix container host passes it through together with the gid that owns it.
+# This script therefore only verifies that it is usable and never modifies host or container state.
+[ -e /dev/kvm ] || die "/dev/kvm is not present. The container must be started with the KVM device passed through."
+
+if [ ! -r /dev/kvm ] || [ ! -w /dev/kvm ]; then
+    die "No read/write access to /dev/kvm as '$(id -un)' (groups: $(id -Gn)). The container needs to be started with the gid owning /dev/kvm."
+fi
+
+emulator_log="${HELIX_WORKITEM_UPLOAD_ROOT:-/tmp}/emulator-$avd_name.log"
+logcat_log="${HELIX_WORKITEM_UPLOAD_ROOT:-/tmp}/emulator-$avd_name-logcat.log"
+
+wait_for_boot() {
+    local deadline=$((SECONDS + boot_timeout_seconds))
+
+    "$adb_binary" start-server > /dev/null 2>&1
+
+    while [ $SECONDS -lt $deadline ]; do
+        if [ -n "$emulator_pid" ] && ! kill -0 "$emulator_pid" 2>/dev/null; then
+            echo "Emulator process exited while booting."
+            return 1
+        fi
+
+        if [ "$("$adb_binary" shell getprop sys.boot_completed 2>/dev/null | tr -d '\r\n')" = "1" ]; then
+            return 0
+        fi
+
+        sleep 2
+    done
+
+    echo "Emulator did not finish booting within $boot_timeout_seconds seconds."
+    return 1
+}
+
+emulator_pid=""
+
+for attempt in $(seq 1 "$boot_attempts"); do
+    echo "Starting AVD '$avd_name' (attempt $attempt of $boot_attempts)."
+
+    # Each container starts from a pristine image, so the emulator does not need '-wipe-data' the
+    # way the shared per-machine emulator did, and skipping it saves boot time.
+    "$emulator_binary" -avd "$avd_name" \
+        -no-window \
+        -no-audio \
+        -no-boot-anim \
+        -no-snapshot \
+        -gpu swiftshader_indirect \
+        -memory "${ANDROID_EMULATOR_MEMORY:-3072}" \
+        -partition-size "${ANDROID_EMULATOR_PARTITION_SIZE:-1550}" \
+        -skip-adb-auth \
+        -detect-image-hang \
+        -restart-when-stalled \
+        -logcat-output "$logcat_log" \
+        -prop net.ipv6.conf.all.disable_ipv6=0 \
+        -prop net.ipv6.conf.lo.disable_ipv6=0 \
+        > "$emulator_log" 2>&1 &
+    emulator_pid=$!
+
+    if wait_for_boot; then
+        echo "AVD '$avd_name' booted (pid $emulator_pid)."
+
+        if [ -n "${HELIX_WORKITEM_ROOT:-}" ]; then
+            echo "$emulator_pid" > "$HELIX_WORKITEM_ROOT/.emulator-pid"
+        fi
+
+        # XHarness installs unsigned/debug APKs, which the package verifier rejects.
+        "$adb_binary" shell settings put global verifier_verify_adb_installs 0 > /dev/null 2>&1
+        "$adb_binary" shell settings put global package_verifier_enable 0 > /dev/null 2>&1
+
+        "$adb_binary" devices -l
+        exit 0
+    fi
+
+    echo "Emulator failed to boot; tail of $emulator_log:"
+    tail -n 30 "$emulator_log" 2>/dev/null
+
+    kill "$emulator_pid" 2>/dev/null
+    wait "$emulator_pid" 2>/dev/null
+    "$adb_binary" kill-server > /dev/null 2>&1
+    emulator_pid=""
+done
+
+die "AVD '$avd_name' failed to boot after $boot_attempts attempts."

eng/testing/Android/stop-emulator.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+# Shuts down the emulator started by start-emulator.sh, as a Helix post-command.
+#
+# Best effort by design: the work item result has already been decided by the time this runs, so a
+# failure to shut down cleanly must not turn a passing run into a failing one. The container is torn
+# down immediately afterwards anyway; this exists so the emulator gets a chance to flush its logs.
+
+set -uo pipefail
+
+android_sdk_root="${ANDROID_SDK_ROOT:-${ANDROID_HOME:-/usr/local/android-sdk}}"
+adb_binary="$android_sdk_root/platform-tools/adb"
+shutdown_timeout_seconds="${ANDROID_EMULATOR_SHUTDOWN_TIMEOUT:-30}"
+
+if [ ! -x "$adb_binary" ]; then
+    echo "adb not found at '$adb_binary'; nothing to stop."
+    exit 0
+fi
+
+emulator_pid=""
+if [ -n "${HELIX_WORKITEM_ROOT:-}" ] && [ -f "$HELIX_WORKITEM_ROOT/.emulator-pid" ]; then
+    emulator_pid="$(cat "$HELIX_WORKITEM_ROOT/.emulator-pid")"
+fi
+
+# start-emulator.sh only records a pid when it started the emulator itself. On the legacy queues the
+# emulator belongs to the machine and is shared by later work items, so it must be left alone.
+if [ -z "$emulator_pid" ]; then
+    echo "This work item did not start the emulator; leaving it running."
+    exit 0
+fi
+
+echo "Stopping Android emulator."
+"$adb_binary" emu kill > /dev/null 2>&1
+
+if [ -n "$emulator_pid" ]; then
+    deadline=$((SECONDS + shutdown_timeout_seconds))
+    while [ $SECONDS -lt $deadline ] && kill -0 "$emulator_pid" 2>/dev/null; do
+        sleep 1
+    done
+
+    if kill -0 "$emulator_pid" 2>/dev/null; then
+        echo "Emulator (pid $emulator_pid) did not exit within $shutdown_timeout_seconds seconds; killing it."
+        kill -9 "$emulator_pid" 2>/dev/null
+    fi
+fi
+
+"$adb_binary" kill-server > /dev/null 2>&1
+
+exit 0

src/libraries/sendtohelix-mobile.targets

@@ -40,6 +40,8 @@
     <HelixPreCommand Include="export XHARNESS_LOG_WITH_TIMESTAMPS=true" />
   </ItemGroup>
 
+  <Import Project="$(RepoRoot)eng/testing/Android/helix.targets" />
+
   <Target Name="PrepareForBuildHelixWorkItems_Mobile">
 
     <PropertyGroup Condition="'$(TargetsAppleMobile)' == 'true'">

src/tests/Common/helixpublishwitharcade.proj

@@ -774,6 +774,8 @@
     <_WorkaroundForNuGetMigrationsForPrepending>$(_WorkaroundForNuGetMigrations) &amp;&amp;</_WorkaroundForNuGetMigrationsForPrepending>
   </PropertyGroup>
 
+  <Import Project="$(RepoRoot)eng/testing/Android/helix.targets" />
+
   <PropertyGroup>
     <HelixPreCommands>@(HelixPreCommand)</HelixPreCommands>
     <HelixPostCommands>@(HelixPostCommand)</HelixPostCommands>