pythongh-116622: Add Android testbed (pythonGH-117878)

Add code and config for a minimal Android app, and instructions to build and run it.
Improve Android build instructions in general.
Add a tool subcommand to download the Gradle wrapper (with its binary blob). Android
studio must be downloaded manually (due to the license).

Author: mhsmith

.github/CODEOWNERS

@@ -243,6 +243,18 @@ Lib/test/support/interpreters/  @ericsnowcurrently
 Modules/_xx*interp*module.c   @ericsnowcurrently
 Lib/test/test_interpreters/   @ericsnowcurrently
 
+# Android
+**/*Android*                  @mhsmith
+**/*android*                  @mhsmith
+
+# iOS (but not termios)
+**/iOS*                       @freakboy3742
+**/ios*                       @freakboy3742
+**/*_iOS*                     @freakboy3742
+**/*_ios*                     @freakboy3742
+**/*-iOS*                     @freakboy3742
+**/*-ios*                     @freakboy3742
+
 # WebAssembly
 /Tools/wasm/                  @brettcannon
 

Android/README.md

@@ -22,12 +22,25 @@ you don't already have the SDK, here's how to install it:
   `android-sdk/cmdline-tools/latest`.
 * `export ANDROID_HOME=/path/to/android-sdk`
 
+The `android.py` script also requires the following commands to be on the `PATH`:
+
+* `curl`
+* `java`
+* `tar`
+* `unzip`
+
 
 ## Building
 
-Building for Android requires doing a cross-build where you have a "build"
-Python to help produce an Android build of CPython. This procedure has been
-tested on Linux and macOS.
+Python can be built for Android on any POSIX platform supported by the Android
+development tools, which currently means Linux or macOS. This involves doing a
+cross-build where you use a "build" Python (for your development machine) to
+help produce a "host" Python for Android.
+
+First, make sure you have all the usual tools and libraries needed to build
+Python for your development machine. The only Android tool you need to install
+is the command line tools package above: the build script will download the
+rest.
 
 The easiest way to do a build is to use the `android.py` script. You can either
 have it perform the entire build process from start to finish in one step, or
@@ -43,9 +56,10 @@ The discrete steps for building via `android.py` are:
 ./android.py make-host HOST
 ```
 
-To see the possible values of HOST, run `./android.py configure-host --help`.
+`HOST` identifies which architecture to build. To see the possible values, run
+`./android.py configure-host --help`.
 
-Or to do it all in a single command, run:
+To do all steps in a single command, run:
 
 ```sh
 ./android.py build HOST
@@ -62,3 +76,22 @@ call. For example, if you want a pydebug build that also caches the results from
 ```sh
 ./android.py build HOST -- -C --with-pydebug
 ```
+
+
+## Testing
+
+To run the Python test suite on Android:
+
+* Install Android Studio, if you don't already have it.
+* Follow the instructions in the previous section to build all supported
+  architectures.
+* Run `./android.py setup-testbed` to download the Gradle wrapper.
+* Open the `testbed` directory in Android Studio.
+* In the *Device Manager* dock, connect a device or start an emulator.
+  Then select it from the drop-down list in the toolbar.
+* Click the "Run" button in the toolbar.
+* The testbed app displays nothing on screen while running. To see its output,
+  open the [Logcat window](https://developer.android.com/studio/debug/logcat).
+
+To run specific tests, or pass any other arguments to the test suite, edit the
+command line in testbed/app/src/main/python/main.py.

Android/android.py

@@ -7,8 +7,9 @@
 import subprocess
 import sys
 import sysconfig
-from os.path import relpath
+from os.path import basename, relpath
 from pathlib import Path
+from tempfile import TemporaryDirectory
 
 SCRIPT_NAME = Path(__file__).name
 CHECKOUT = Path(__file__).resolve().parent.parent
@@ -102,11 +103,17 @@ def unpack_deps(host):
     for name_ver in ["bzip2-1.0.8-1", "libffi-3.4.4-2", "openssl-3.0.13-1",
                      "sqlite-3.45.1-0", "xz-5.4.6-0"]:
         filename = f"{name_ver}-{host}.tar.gz"
-        run(["wget", f"{deps_url}/{name_ver}/{filename}"])
+        download(f"{deps_url}/{name_ver}/{filename}")
         run(["tar", "-xf", filename])
         os.remove(filename)
 
 
+def download(url, target_dir="."):
+    out_path = f"{target_dir}/{basename(url)}"
+    run(["curl", "-Lf", "-o", out_path, url])
+    return out_path
+
+
 def configure_host_python(context):
     host_dir = subdir(context.host, clean=context.clean)
 
@@ -160,6 +167,30 @@ def clean_all(context):
     delete_if_exists(CROSS_BUILD_DIR)
 
 
+# To avoid distributing compiled artifacts without corresponding source code,
+# the Gradle wrapper is not included in the CPython repository. Instead, we
+# extract it from the Gradle release.
+def setup_testbed(context):
+    ver_long = "8.7.0"
+    ver_short = ver_long.removesuffix(".0")
+    testbed_dir = CHECKOUT / "Android/testbed"
+
+    for filename in ["gradlew", "gradlew.bat"]:
+        out_path = download(
+            f"https://raw.githubusercontent.com/gradle/gradle/v{ver_long}/{filename}",
+            testbed_dir)
+        os.chmod(out_path, 0o755)
+
+    with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir:
+        os.chdir(temp_dir)
+        bin_zip = download(
+            f"https://services.gradle.org/distributions/gradle-{ver_short}-bin.zip")
+        outer_jar = f"gradle-{ver_short}/lib/plugins/gradle-wrapper-{ver_short}.jar"
+        run(["unzip", bin_zip, outer_jar])
+        run(["unzip", "-o", "-d", f"{testbed_dir}/gradle/wrapper", outer_jar,
+             "gradle-wrapper.jar"])
+
+
 def main():
     parser = argparse.ArgumentParser()
     subcommands = parser.add_subparsers(dest="subcommand")
@@ -173,8 +204,11 @@ def main():
                                             help="Run `configure` for Android")
     make_host = subcommands.add_parser("make-host",
                                        help="Run `make` for Android")
-    clean = subcommands.add_parser("clean", help="Delete files and directories "
-                                                 "created by this script")
+    subcommands.add_parser(
+        "clean", help="Delete the cross-build directory")
+    subcommands.add_parser(
+        "setup-testbed", help="Download the testbed Gradle wrapper")
+
     for subcommand in build, configure_build, configure_host:
         subcommand.add_argument(
             "--clean", action="store_true", default=False, dest="clean",
@@ -194,7 +228,8 @@ def main():
                 "configure-host": configure_host_python,
                 "make-host": make_host_python,
                 "build": build_all,
-                "clean": clean_all}
+                "clean": clean_all,
+                "setup-testbed": setup_testbed}
     dispatch[context.subcommand](context)
 
 

Android/testbed/.gitignore

@@ -0,0 +1,21 @@
+# The Gradle wrapper should be downloaded by running `../android.py setup-testbed`.
+/gradlew
+/gradlew.bat
+/gradle/wrapper/gradle-wrapper.jar
+
+*.iml
+.gradle
+/local.properties
+/.idea/caches
+/.idea/deploymentTargetDropdown.xml
+/.idea/libraries
+/.idea/modules.xml
+/.idea/workspace.xml
+/.idea/navEditor.xml
+/.idea/assetWizardSettings.xml
+.DS_Store
+/build
+/captures
+.externalNativeBuild
+.cxx
+local.properties

Android/testbed/app/.gitignore

@@ -0,0 +1 @@
+/build

Android/testbed/app/build.gradle.kts

@@ -0,0 +1,129 @@
+import com.android.build.api.variant.*
+
+plugins {
+    id("com.android.application")
+    id("org.jetbrains.kotlin.android")
+}
+
+val PYTHON_DIR = File(projectDir, "../../..").canonicalPath
+val PYTHON_CROSS_DIR = "$PYTHON_DIR/cross-build"
+val ABIS = mapOf(
+    "arm64-v8a" to "aarch64-linux-android",
+    "x86_64" to "x86_64-linux-android",
+)
+
+val PYTHON_VERSION = File("$PYTHON_DIR/Include/patchlevel.h").useLines {
+    for (line in it) {
+        val match = """#define PY_VERSION\s+"(\d+\.\d+)""".toRegex().find(line)
+        if (match != null) {
+            return@useLines match.groupValues[1]
+        }
+    }
+    throw GradleException("Failed to find Python version")
+}
+
+
+android {
+    namespace = "org.python.testbed"
+    compileSdk = 34
+
+    defaultConfig {
+        applicationId = "org.python.testbed"
+        minSdk = 21
+        targetSdk = 34
+        versionCode = 1
+        versionName = "1.0"
+
+        ndk.abiFilters.addAll(ABIS.keys)
+        externalNativeBuild.cmake.arguments(
+            "-DPYTHON_CROSS_DIR=$PYTHON_CROSS_DIR",
+            "-DPYTHON_VERSION=$PYTHON_VERSION")
+    }
+
+    externalNativeBuild.cmake {
+        path("src/main/c/CMakeLists.txt")
+    }
+
+    // Set this property to something non-empty, otherwise it'll use the default
+    // list, which ignores asset directories beginning with an underscore.
+    aaptOptions.ignoreAssetsPattern = ".git"
+
+    compileOptions {
+        sourceCompatibility = JavaVersion.VERSION_1_8
+        targetCompatibility = JavaVersion.VERSION_1_8
+    }
+    kotlinOptions {
+        jvmTarget = "1.8"
+    }
+}
+
+dependencies {
+    implementation("androidx.appcompat:appcompat:1.6.1")
+    implementation("com.google.android.material:material:1.11.0")
+    implementation("androidx.constraintlayout:constraintlayout:2.1.4")
+}
+
+
+// Create some custom tasks to copy Python and its standard library from
+// elsewhere in the repository.
+androidComponents.onVariants { variant ->
+    generateTask(variant, variant.sources.assets!!) {
+        into("python") {
+            for (triplet in ABIS.values) {
+                for (subDir in listOf("include", "lib")) {
+                    into(subDir) {
+                        from("$PYTHON_CROSS_DIR/$triplet/prefix/$subDir")
+                        include("python$PYTHON_VERSION/**")
+                        duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+                    }
+                }
+            }
+            into("lib/python$PYTHON_VERSION") {
+                // Uncomment this to pick up edits from the source directory
+                // without having to rerun `make install`.
+                // from("$PYTHON_DIR/Lib")
+                // duplicatesStrategy = DuplicatesStrategy.INCLUDE
+
+                into("site-packages") {
+                    from("$projectDir/src/main/python")
+                }
+            }
+        }
+        exclude("**/__pycache__")
+    }
+
+    generateTask(variant, variant.sources.jniLibs!!) {
+        for ((abi, triplet) in ABIS.entries) {
+            into(abi) {
+                from("$PYTHON_CROSS_DIR/$triplet/prefix/lib")
+                include("libpython*.*.so")
+                include("lib*_python.so")
+            }
+        }
+    }
+}
+
+
+fun generateTask(
+    variant: ApplicationVariant, directories: SourceDirectories,
+    configure: GenerateTask.() -> Unit
+) {
+    val taskName = "generate" +
+        listOf(variant.name, "Python", directories.name)
+            .map { it.replaceFirstChar(Char::uppercase) }
+            .joinToString("")
+
+    directories.addGeneratedSourceDirectory(
+        tasks.register<GenerateTask>(taskName) {
+            into(outputDir)
+            configure()
+        },
+        GenerateTask::outputDir)
+}
+
+
+// addGeneratedSourceDirectory requires the task to have a DirectoryProperty.
+abstract class GenerateTask: Sync() {
+    @get:OutputDirectory
+    abstract val outputDir: DirectoryProperty
+}

Android/testbed/app/src/main/AndroidManifest.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+
+    <uses-permission android:name="android.permission.INTERNET"/>
+
+    <application
+        android:icon="@drawable/ic_launcher"
+        android:label="@string/app_name"
+        android:theme="@style/Theme.Material3.Light.NoActionBar">
+        <activity
+            android:name=".MainActivity"
+            android:exported="true">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+        </activity>
+    </application>
+
+</manifest>

Android/testbed/app/src/main/c/CMakeLists.txt

@@ -0,0 +1,9 @@
+cmake_minimum_required(VERSION 3.4.1)
+project(testbed)
+
+set(PREFIX_DIR ${PYTHON_CROSS_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/prefix)
+include_directories(${PREFIX_DIR}/include/python${PYTHON_VERSION})
+link_directories(${PREFIX_DIR}/lib)
+link_libraries(log python${PYTHON_VERSION})
+
+add_library(main_activity SHARED main_activity.c)