ci: Run tests against Android NDK

.github/workflows/images.yml

@@ -37,6 +37,7 @@ jobs:
       fail-fast: false
       matrix:
         cfg:
+          - { name: Android NDK,                  id: android        }
           - { name: Arch Linux,                   id: arch           }
           - { name: CUDA (on Arch),               id: cuda           }
           - { name: CUDA Cross (on Ubuntu Jammy), id: cuda-cross     }

.github/workflows/nonnative.yml

@@ -49,3 +49,16 @@ jobs:
     - uses: actions/checkout@v4
     - name: Run tests
       run: bash -c 'source /ci/env_vars.sh; cd $GITHUB_WORKSPACE; ./run_tests.py $CI_ARGS --cross cuda-cross.json --cross-only'
+
+  cross-android:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        cfg:
+          - { platform: android, arch: aarch64 }
+          - { platform: android, arch: x86_64 }
+    container: mesonbuild/android:latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Run tests
+      run: bash -c 'source /ci/env_vars.sh; cd $GITHUB_WORKSPACE; ./run_tests.py --cross /opt/android/meson/android-${ANDROID_NDKVER}-${{ matrix.cfg.platform }}${ANDROID_TARGET}-${{ matrix.cfg.arch }}-cross.json --cross-only'

ci/ciimage/android/image.json

@@ -0,0 +1,19 @@
+{
+  "base_image": "debian:sid",
+  "args": [
+    "--cross", "/opt/android/meson/android-${ANDROID_NDKVER}-androideabi${ANDROID_TARGET}-armv7a-cross.json",
+    "--cross", "/opt/android/meson/android-${ANDROID_NDKVER}-android${ANDROID_TARGET}-aarch64-cross.json",
+    "--cross", "/opt/android/meson/android-${ANDROID_NDKVER}-android${ANDROID_TARGET}-i686-cross.json",
+    "--cross", "/opt/android/meson/android-${ANDROID_NDKVER}-android${ANDROID_TARGET}-x86_64-cross.json",
+    "--cross", "/opt/android/meson/android-${ANDROID_NDKVER}-android35-riscv64-cross.txt"
+  ],
+  "env": {
+    "ANDROID_HOME": "/opt/android",
+    "ANDROID_SDKVER": "36.1.0",
+    "ANDROID_NDKVER": "29.0.14206865",
+    "ANDROID_TARGET": "24",
+    "CI": "1",
+    "MESON_CI_JOBNAME": "android-cross"
+  },
+  "needs_meson_in_install": true
+}

ci/ciimage/android/install.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+
+set -e
+
+source /ci/common.sh
+source /ci/env_vars.sh
+
+export DEBIAN_FRONTEND=noninteractive
+export LANG='C.UTF-8'
+
+apt-get -y update
+apt-get -y upgrade
+
+pkgs=(
+  git jq ninja-build python3-pip sdkmanager
+)
+
+apt-get -y install "${pkgs[@]}"
+
+install_minimal_python_packages
+
+# cleanup
+apt-get -y clean
+apt-get -y autoclean
+
+# sdk install
+
+set -x
+
+if [[ -z $ANDROID_HOME || -z $ANDROID_SDKVER || -z $ANDROID_NDKVER ]]; then
+    echo "ANDROID_HOME, ANDROID_SDKVER and ANDROID_NDKVER env var must be set!"
+    exit 1
+fi
+
+mkdir -p ${HOME}/.android
+# there are currently zero user repos
+echo 'count=0' > ${HOME}/.android/repositories.cfg
+cat <<EOF >> ${HOME}/.android/sites-settings.cfg
+@version@=1
+@disabled@https\://dl.google.com/android/repository/extras/intel/addon.xml=disabled
+@disabled@https\://dl.google.com/android/repository/glass/addon.xml=disabled
+@disabled@https\://dl.google.com/android/repository/sys-img/android/sys-img.xml=disabled
+@disabled@https\://dl.google.com/android/repository/sys-img/android-tv/sys-img.xml=disabled
+@disabled@https\://dl.google.com/android/repository/sys-img/android-wear/sys-img.xml=disabled
+@disabled@https\://dl.google.com/android/repository/sys-img/google_apis/sys-img.xml=disabled
+EOF
+
+ANDROID_SDKMAJOR=${ANDROID_SDKVER%%.*}
+
+# accepted licenses
+
+mkdir -p $ANDROID_HOME/licenses/
+
+cat << EOF > $ANDROID_HOME/licenses/android-sdk-license
+
+8933bad161af4178b1185d1a37fbf41ea5269c55
+
+d56f5187479451eabf01fb78af6dfcb131a6481e
+
+24333f8a63b6825ea9c5514f83c2829b004d1fee
+EOF
+
+cat <<EOF > $ANDROID_HOME/licenses/android-sdk-preview-license
+
+84831b9409646a918e30573bab4c9c91346d8abd
+EOF
+
+cat <<EOF > $ANDROID_HOME/licenses/android-sdk-preview-license-old
+
+79120722343a6f314e0719f863036c702b0e6b2a
+
+84831b9409646a918e30573bab4c9c91346d8abd
+EOF
+
+cat <<EOF > $ANDROID_HOME/licenses/intel-android-extra-license
+
+d975f751698a77b662f1254ddbeed3901e976f5a
+EOF
+
+sdkmanager --sdk_root "${ANDROID_HOME}" \
+	"ndk;${ANDROID_NDKVER}"
+
+kernel=$(uname -s)
+arch=$(uname -m)
+
+tee "${ANDROID_HOME}/toolchain.cross" <<EOF
+[constants]
+toolchain='${ANDROID_HOME}/ndk/${ANDROID_NDKVER}/toolchains/llvm/prebuilt/${kernel,,}-${arch}'
+EOF
+
+/meson_private/meson.py env2mfile --android -o "${ANDROID_HOME}/meson/"
+find "${ANDROID_HOME}/meson/" -exec sh -c 'for cf; do jq -nr --arg file "$cf" "{ \"file\": \$file, \"env\": [], \"tests\": [\"common\", \"failing-meson\", \"failing-build\", \"platform-android\"] }" > "${cf%%.txt}.json"; done' sh {} +

ci/ciimage/build.py

@@ -28,9 +28,11 @@ def __init__(self, image_dir: Path) -> None:
         self.base_image: str = data['base_image']
         self.args: T.List[str] = data.get('args', [])
         self.env: T.Dict[str, str] = data['env']
+        self.needs_meson_in_install = data.get('needs_meson_in_install', False)
 
 class BuilderBase():
     def __init__(self, data_dir: Path, temp_dir: Path) -> None:
+        self.meson_root = data_dir.parent.parent.parent.resolve()
         self.data_dir = data_dir
         self.temp_dir = temp_dir
 
@@ -60,6 +62,20 @@ def validate_data_dir(self) -> None:
             if not i.is_file():
                 raise RuntimeError(f'{i.as_posix()} is not a regular file')
 
+    def copy_meson(self) -> None:
+        shutil.copytree(
+            self.meson_root,
+            self.temp_dir / 'meson',
+            symlinks=True,
+            ignore=shutil.ignore_patterns(
+                '.git',
+                '*_cache',
+                '__pycache__',
+                # 'work area',
+                self.temp_dir.name,
+            ),
+        )
+
 class Builder(BuilderBase):
     def gen_bashrc(self) -> None:
         out_file = self.temp_dir / 'env_vars.sh'
@@ -91,6 +107,8 @@ def gen_dockerfile(self) -> None:
         out_data = textwrap.dedent(f'''\
             FROM {self.image_def.base_image}
 
+            { "ADD meson /meson_private" if self.image_def.needs_meson_in_install else "" }
+
             ADD install.sh  /ci/install.sh
             ADD common.sh   /ci/common.sh
             ADD env_vars.sh /ci/env_vars.sh
@@ -105,6 +123,9 @@ def do_build(self) -> None:
             shutil.copy(str(i), str(self.temp_dir))
         shutil.copy(str(self.common_sh), str(self.temp_dir))
 
+        if self.image_def.needs_meson_in_install:
+            self.copy_meson()
+
         self.gen_bashrc()
         self.gen_dockerfile()
 
@@ -139,20 +160,6 @@ def gen_dockerfile(self) -> None:
 
         out_file.write_text(out_data, encoding='utf-8')
 
-    def copy_meson(self) -> None:
-        shutil.copytree(
-            self.meson_root,
-            self.temp_dir / 'meson',
-            symlinks=True,
-            ignore=shutil.ignore_patterns(
-                '.git',
-                '*_cache',
-                '__pycache__',
-                # 'work area',
-                self.temp_dir.name,
-            ),
-        )
-
     def do_test(self, tty: bool = False) -> None:
         self.copy_meson()
         self.gen_dockerfile()

mesonbuild/scripts/env2mfile.py

@@ -464,15 +464,18 @@ def __init__(self, options: T.Any):
         self.outdir = pathlib.Path(options.outfile)
 
     def detect_android_sdk_root(self) -> None:
-        home = pathlib.Path.home()
-        if self.platform == 'windows':
-            sdk_root = home / 'AppData/Local/Android/Sdk'
-        elif self.platform == 'darwin':
-            sdk_root = home / 'Library/Android/Sdk'
-        elif self.platform == 'linux':
-            sdk_root = home / 'Android/Sdk'
-        else:
-            sys.exit('Unsupported platform.')
+        android_home = os.getenv('ANDROID_HOME')
+        sdk_root = None if android_home is None else pathlib.Path(android_home)
+        if sdk_root is None:
+            home = pathlib.Path.home()
+            if self.platform == 'windows':
+                sdk_root = home / 'AppData/Local/Android/Sdk'
+            elif self.platform == 'darwin':
+                sdk_root = home / 'Library/Android/Sdk'
+            elif self.platform == 'linux':
+                sdk_root = home / 'Android/Sdk'
+            else:
+                sys.exit('Unsupported platform.')
         if not sdk_root.is_dir():
             sys.exit(f'Could not locate Android SDK root in {sdk_root}.')
         ndk_root = sdk_root / 'ndk'
@@ -495,14 +498,21 @@ def process_ndk(self, ndk: pathlib.Path) -> None:
         bindir = toolchain_root / 'bin'
         if not bindir.is_dir():
             sys.exit(f'Could not detect toolchain in {toolchain_root}.')
-        ar_path = bindir / f'llvm-ar{self.exe_suffix}'
-        if not ar_path.is_file():
-            sys.exit(f'Could not detect llvm-ar in {toolchain_root}.')
-        ar_str = str(ar_path).replace('\\', '/')
-        strip_path = bindir / f'llvm-strip{self.exe_suffix}'
-        if not strip_path.is_file():
-            sys.exit(f'Could not detect llvm-strip n {toolchain_root}.')
-        strip_str = str(strip_path).replace('\\', '/')
+
+        bin_tools = {
+            'ar': 'llvm-ar',
+            'as': 'llvm-as',
+            'ranlib': 'llvm-ranlib',
+            'ld': 'ld',
+            'strip': 'llvm-strip',
+        }
+        bin_mappings = {}
+        for name, tool in bin_tools.items():
+            path = bindir / f'{tool}{self.exe_suffix}'
+            if not path.is_file():
+                sys.exit(f'Could not detect {tool} in {toolchain_root}.')
+            bin_mappings[name] = str(path).replace('\\', '/')
+
         for compiler in bindir.glob('*-clang++'):
             parts = compiler.parts[-1].split('-')
             assert len(parts) == 4
@@ -518,8 +528,14 @@ def process_ndk(self, ndk: pathlib.Path) -> None:
                 ofile.write('[binaries]\n')
                 ofile.write(f"c = '{c_compiler_str}'\n")
                 ofile.write(f"cpp = '{cpp_compiler_str}'\n")
-                ofile.write(f"ar = '{ar_str}'\n")
-                ofile.write(f"strip = '{strip_str}'\n")
+                for name, path in bin_mappings.items():
+                    ofile.write(f"{name} = '{path}'\n")
+
+                ofile.write('\n[built-in options]\n')
+                ofile.write('c_args        = []\n')
+                ofile.write("c_link_args   = ['-Wl,--build-id=sha1']\n")
+                ofile.write('cpp_args      = c_args\n')
+                ofile.write('cpp_link_args = c_link_args\n')
 
                 ofile.write('\n[host_machine]\n')
                 ofile.write("system = 'android'\n")

run_project_tests.py

@@ -39,6 +39,8 @@
 from mesonbuild import mtest
 from mesonbuild.compilers import compiler_from_language
 from mesonbuild.build import ConfigurationData
+from mesonbuild.envconfig import MachineInfo, detect_machine_info
+from mesonbuild.machinefile import parse_machine_files
 from mesonbuild.mesonlib import MachineChoice, Popen_safe, TemporaryDirectoryWinProof, setup_vsenv
 from mesonbuild.mlog import blue, bold, cyan, green, red, yellow, normal_green
 from mesonbuild.coredata import version as meson_version
@@ -605,17 +607,21 @@ def format_parameter_file(file_basename: str, test: TestDef, test_build_dir: str
 
     return destination
 
-def detect_parameter_files(test: TestDef, test_build_dir: str) -> T.Tuple[Path, Path]:
+def detect_parameter_files(test: TestDef, test_build_dir: str) -> T.Tuple[Path, Path, Path]:
     nativefile = test.path / 'nativefile.ini'
     crossfile = test.path / 'crossfile.ini'
+    optionsfile = test.path / 'optionsfile.ini'
 
     if os.path.exists(str(test.path / 'nativefile.ini.in')):
         nativefile = format_parameter_file('nativefile.ini', test, test_build_dir)
 
     if os.path.exists(str(test.path / 'crossfile.ini.in')):
         crossfile = format_parameter_file('crossfile.ini', test, test_build_dir)
 
-    return nativefile, crossfile
+    if os.path.exists(str(test.path / 'optionsfile.ini.in')):
+        optionsfile = format_parameter_file('optionsfile.ini', test, test_build_dir)
+
+    return nativefile, crossfile, optionsfile
 
 # In previous python versions the global variables are lost in ProcessPoolExecutor.
 # So, we use this tuple to restore some of them
@@ -671,12 +677,14 @@ def _run_test(test: TestDef,
         gen_args += ['--libdir', 'lib']
     gen_args += [test.path.as_posix(), test_build_dir] + backend_flags + extra_args
 
-    nativefile, crossfile = detect_parameter_files(test, test_build_dir)
+    nativefile, crossfile, optionsfile = detect_parameter_files(test, test_build_dir)
 
     if nativefile.exists():
         gen_args.extend(['--native-file', nativefile.as_posix()])
     if crossfile.exists():
         gen_args.extend(['--cross-file', crossfile.as_posix()])
+    if optionsfile.exists():
+        gen_args.extend(['--cross-file' if '--cross-file' in extra_args else '--native-file', optionsfile.as_posix()])
     inprocess, res = run_configure(gen_args, env=test.env, catch_exception=True)
     returncode, stdo, stde = res
     cmd = '(inprocess) $ ' if inprocess else '$ '
@@ -1079,7 +1087,7 @@ def should_skip_wayland() -> bool:
         return True
     return False
 
-def detect_tests_to_run(only: T.Dict[str, T.List[str]], use_tmp: bool) -> T.List[T.Tuple[str, T.List[TestDef], bool]]:
+def detect_tests_to_run(only: T.Dict[str, T.List[str]], use_tmp: bool, host_machine: MachineInfo) -> T.List[T.Tuple[str, T.List[TestDef], bool]]:
     """
     Parameters
     ----------
@@ -1123,8 +1131,7 @@ def __init__(self, category: str, subdir: str, skip: bool = False, stdout_mandat
         TestCategory('platform-osx', 'osx', not mesonlib.is_osx()),
         TestCategory('platform-windows', 'windows', not mesonlib.is_windows() and not mesonlib.is_cygwin()),
         TestCategory('platform-linux', 'linuxlike', mesonlib.is_osx() or mesonlib.is_windows()),
-        # FIXME, does not actually run in CI, change to run the test if an Android cross toolchain is detected.
-        TestCategory('platform-android', 'android', not mesonlib.is_android()),
+        TestCategory('platform-android', 'android', not host_machine.is_android()),
         TestCategory('java', 'java', backend is not Backend.ninja or not have_java()),
         TestCategory('C#', 'csharp', skip_csharp(backend)),
         TestCategory('vala', 'vala', backend is not Backend.ninja or not shutil.which(os.environ.get('VALAC', 'valac'))),
@@ -1689,6 +1696,13 @@ def setup_symlinks() -> None:
     script_dir = os.path.split(__file__)[0]
     if script_dir != '':
         os.chdir(script_dir)
+
+    if options.cross_file is not None:
+        config = parse_machine_files([options.cross_file], script_dir)
+        host_machine = MachineInfo.from_literal(config['host_machine']) if 'host_machine' in config else detect_machine_info()
+    else:
+        host_machine = detect_machine_info()
+
     check_meson_commands_work(options.use_tmpdir, options.extra_args)
     only = collections.defaultdict(list)
     for i in options.only:
@@ -1698,7 +1712,7 @@ def setup_symlinks() -> None:
         except ValueError:
             only[i].append('')
     try:
-        all_tests = detect_tests_to_run(only, options.use_tmpdir)
+        all_tests = detect_tests_to_run(only, options.use_tmpdir, host_machine)
         res = run_tests(all_tests, 'meson-test-run', options.failfast, options.extra_args, options.use_tmpdir, options.num_workers)
         (passing_tests, failing_tests, skipped_tests) = res
     except StopException:

run_shell_checks.py

@@ -27,6 +27,7 @@
     'ci/ciimage/fedora/install.sh',
     'ci/ciimage/arch/install.sh',
     'ci/ciimage/gentoo/install.sh',
+    'ci/ciimage/android/install.sh',
     'manual tests/4 standalone binaries/myapp.sh',
     'manual tests/4 standalone binaries/osx_bundler.sh',
     'manual tests/4 standalone binaries/linux_bundler.sh',

run_tests.py

@@ -395,7 +395,7 @@ def main():
         for cf in options.cross:
             print(mlog.bold(f'Running {cf} cross tests.'))
             print(flush=True)
-            cmd = cross_test_args + ['cross/' + cf]
+            cmd = cross_test_args + [cf] if cf.startswith("/") else ['cross/' + cf]
             if options.failfast:
                 cmd += ['--failfast']
             if options.cross_only: