Migrate Android Java E2E tests from App Center to Browserstack

tools/ci_build/github/azure-pipelines/templates/android-java-api-aar-test.yml

@@ -45,8 +45,6 @@ jobs:
 
   - template: use-android-ndk.yml
 
-  - template: install-appcenter.yml
-
   - template: use-android-emulator.yml
     parameters:
       create: true
@@ -68,19 +66,21 @@ jobs:
     parameters:
       stop: true
 
+  # we run e2e tests on one older device (Pixel 3) and one newer device (Galaxy 23)
   - script: |
       set -e -x
-      cd android_test/android
-      appcenter test run espresso \
-        --app "AI-Frameworks/ORT-Mobile-Android" \
-        --devices $(app_center_android_test_devices) \
-        --app-path ./app/build/outputs/apk/debug/app-debug.apk \
-        --test-series "master" \
-        --locale "en_US" \
-        --build-dir ./app/build/outputs/apk/androidTest/debug \
-        --token $(app_center_api_token)
-    displayName: Run E2E tests using App Center
-    workingDirectory: $(Build.BinariesDirectory)
+      pip install requests
+      python $(Build.SourcesDirectory)/tools/python/upload_and_run_browserstack_tests.py \
+        --test_platform espresso \
+        --app_apk_path "debug/app-debug.apk" \
+        --test_apk_path "androidTest/debug/app-debug-androidTest.apk" \
+        --devices "Samsung Galaxy S23-13.0" "Google Pixel 3-9.0"
+    displayName: Run E2E tests using Browserstack
+    workingDirectory: $(Build.BinariesDirectory)/android_test/android/app/build/outputs/apk
+    timeoutInMinutes: 15
+    env:
+      BROWSERSTACK_ID: $(browserstack_username)
+      BROWSERSTACK_TOKEN: $(browserstack_access_key)
 
   - template: component-governance-component-detection-steps.yml
     parameters :

tools/python/upload_and_run_browserstack_tests.py

@@ -0,0 +1,158 @@
+import argparse
+import os
+import sys
+import time
+from pathlib import Path
+
+import requests
+
+script_description = """
+After building ONNXRuntime for Android or iOS, use this script to upload the app and test files to BrowserStack then
+run the tests on the specified devices.
+
+Find the Android test app in the repo here (as of rel-1.19.2):
+java/src/test/android
+"""
+
+
+def response_to_json(response):
+    response.raise_for_status()
+    response_json = response.json()
+    print(response_json)
+    print("-" * 30)
+    return response_json
+
+
+def upload_apk_parse_json(post_url, apk_path, id, token):
+    with open(apk_path, "rb") as apk_file:
+        response = requests.post(post_url, files={"file": apk_file}, auth=(id, token), timeout=180)
+    return response_to_json(response)
+
+
+def browserstack_build_request(devices, app_url, test_suite_url, test_platform, id, token):
+    headers = {}
+
+    json_data = {
+        "devices": devices,
+        "app": app_url,
+        "testSuite": test_suite_url,
+    }
+
+    build_response = requests.post(
+        f"https://api-cloud.browserstack.com/app-automate/{test_platform}/v2/build",
+        headers=headers,
+        json=json_data,
+        auth=(id, token),
+        timeout=180,
+    )
+
+    return response_to_json(build_response)
+
+
+def build_query_loop(build_id, test_platform, id, token):
+    """Called after a build is initiated on Browserstack. Repeatedly queries the REST endpoint to check for the
+    status of the tests in 30 second intervals.
+    """
+    tests_status = "running"
+
+    while tests_status == "running":
+        time.sleep(30)
+
+        test_response = requests.get(
+            f"https://api-cloud.browserstack.com/app-automate/{test_platform}/v2/builds/{build_id}",
+            auth=(id, token),
+            timeout=30,
+        )
+
+        test_response_json = response_to_json(test_response)
+        tests_status = test_response_json["status"]
+
+    return tests_status
+
+
+if __name__ == "__main__":
+    # handle cli args
+    parser = argparse.ArgumentParser(script_description)
+
+    parser.add_argument(
+        "--test_platform", type=str, help="Testing platform", choices=["espresso", "xcuitest"], required=True
+    )
+    parser.add_argument(
+        "--app_apk_path",
+        type=Path,
+        help=(
+            "Path to the app APK. "
+            "Typically, the app APK is in "
+            "{build_output_dir}/android_test/android/app/build/outputs/apk/debug/app-debug.apk"
+        ),
+        required=True,
+    )
+    parser.add_argument(
+        "--test_apk_path",
+        type=Path,
+        help=(
+            "Path to the test APK. "
+            "Typically, the test APK is in "
+            "{build_output_dir}/android_test/android/app/build/outputs/apk/androidTest/debug/app-debug-androidTest.apk"
+        ),
+        required=True,
+    )
+    parser.add_argument(
+        "--devices",
+        type=str,
+        nargs="+",
+        help="List of devices to run the tests on. For more info, "
+        "see https://www.browserstack.com/docs/app-automate/espresso/specify-devices",
+        required=True,
+    )
+
+    args = parser.parse_args()
+
+    try:
+        browserstack_id = os.environ["BROWSERSTACK_ID"]
+        browserstack_token = os.environ["BROWSERSTACK_TOKEN"]
+    except KeyError:
+        print("Please set the environment variables BROWSERSTACK_ID and BROWSERSTACK_TOKEN")
+        print(
+            "These values will be found at https://app-automate.browserstack.com/dashboard/v2 & clicking 'ACCESS KEY'"
+        )
+        sys.exit(1)
+
+    # Upload the app and test suites
+    upload_app_json = upload_apk_parse_json(
+        f"https://api-cloud.browserstack.com/app-automate/{args.test_platform}/v2/app",
+        args.app_apk_path,
+        browserstack_id,
+        browserstack_token,
+    )
+    upload_test_json = upload_apk_parse_json(
+        f"https://api-cloud.browserstack.com/app-automate/{args.test_platform}/v2/test-suite",
+        args.test_apk_path,
+        browserstack_id,
+        browserstack_token,
+    )
+
+    # Initiate build (send request to run the tests)
+    build_response_json = browserstack_build_request(
+        args.devices,
+        upload_app_json["app_url"],
+        upload_test_json["test_suite_url"],
+        args.test_platform,
+        browserstack_id,
+        browserstack_token,
+    )
+
+    # Get build status until the tests are no longer running
+    tests_status = build_query_loop(
+        build_response_json["build_id"], args.test_platform, browserstack_id, browserstack_token
+    )
+
+    test_suite_details_url = (
+        f"https://app-automate.browserstack.com/dashboard/v2/builds/{build_response_json['build_id']}"
+    )
+
+    print("=" * 30)
+    print("Test suite details: ", test_suite_details_url)
+    print("=" * 30)
+    if tests_status != "passed":
+        raise Exception(f"Tests failed. Go to {test_suite_details_url} for more details.")