CI: build packages on all arches (MR 802)

Split the aports-build job into arch-specific build-x86_64,
build-aarch64, ... jobs. Extend build_changed_aports.py to accept the
architecture as argument, and to build all packages for that arch where
possible.

Import and use pmbootstrap code for APKBUILD parsing etc, as we do this
already in various testcases running in this repository.

Co-Authored-By: Bart Ribbers <bribbers@disroot.org>

.gitlab-ci.yml

@@ -133,16 +133,31 @@ mr-settings:
     - python3 ./check_mr_settings.py
 
 # build changed aports
-aports-build:
+.build:
   stage: second
   <<: *only-default
   before_script:
     - wget "https://gitlab.com/postmarketOS/ci-common/-/raw/master/install_pmbootstrap.sh"
     - sh ./install_pmbootstrap.sh
   script:
-    - PYTHONUNBUFFERED=1 su pmos -c ".gitlab-ci/build_changed_aports.py"
+    - .gitlab-ci/build.sh
     - cp -r /home/pmos/.local/var/pmbootstrap/packages/ packages/ || true
   artifacts:
     expire_in: 1 week
     paths:
       - packages/
+
+build-x86_64:
+  extends: .build
+
+build-x86:
+  extends: .build
+
+build-aarch64:
+  extends: .build
+
+build-armv7:
+  extends: .build
+
+build-armhf:
+  extends: .build

.gitlab-ci/build.sh

@@ -0,0 +1,8 @@
+#!/bin/sh -e
+# Convenience wrapper for short arch-specific build jobs in .gitlab-ci.yml
+
+export PYTHONUNBUFFERED=1
+JOB_ARCH="${CI_JOB_NAME#build-}"
+
+set -x
+su pmos -c ".gitlab-ci/build_changed_aports.py $JOB_ARCH"

.gitlab-ci/build_changed_aports.py

@@ -1,37 +1,79 @@
 #!/usr/bin/env python3
 # Copyright 2020 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
+import sys
 
 # Same dir
 import common
 
+# pmbootstrap
+import testcases.add_pmbootstrap_to_import_path
+import pmb.parse
+import pmb.parse._apkbuild
+import pmb.helpers.pmaports
 
-def check_build(packages, verify_only=False):
+
-    # Initialize build environment with less logging
+def build_strict(packages, arch):
     common.run_pmbootstrap(["build_init"])
+    common.run_pmbootstrap(["--details-to-stdout", "build", "--strict",
+                            "--force", "--arch", arch] + list(packages))
 
-    if verify_only:
+
-        common.run_pmbootstrap(["--details-to-stdout", "checksum",
+def verify_checksums(packages, arch):
-                                "--verify"] + list(packages))
+    # Only do this with one build-{arch} job
-    else:
+    arch_verify = "x86_64"
-        common.run_pmbootstrap(["--details-to-stdout", "build", "--strict",
+    if arch != arch_verify:
-                                "--force"] + list(packages))
+        print(f"NOTE: doing checksum verification in build-{arch_verify} job,"
+              " not here.")
+        return
+
+    common.run_pmbootstrap(["build_init"])
+    common.run_pmbootstrap(["--details-to-stdout", "checksum", "--verify"] +
+                           list(packages))
 
 
 if __name__ == "__main__":
+    # Architecture to build for (as in build-{arch})
+    if len(sys.argv) != 2:
+        print("usage: build_changed_aports.py ARCH")
+        sys.exit(1)
+    arch = sys.argv[1]
+
     # Get and print modified packages
     common.add_upstream_git_remote()
     packages = common.get_changed_packages()
 
-    # Build changed packages
+    # Package count sanity check
     common.get_changed_packages_sanity_check(len(packages))
+
+    # [ci:skip-build]: verify checksums and stop
+    verify_only = common.commit_message_has_string("[ci:skip-build]")
+    if verify_only:
+        print("WARNING: not building changed packages ([ci:skip-build])!")
+        print("verifying checksums: " + ", ".join(packages))
+        verify_checksums(packages, arch)
+        sys.exit(0)
+
+    # Prepare "args" to use pmbootstrap code
+    sys.argv = ["pmbootstrap", "chroot"]
+    args = pmb.parse.arguments()
+
+    # Filter out packages that can't be built for given arch
+    # (Iterate over copy of packages, because we modify it in this loop)
+    for package in packages.copy():
+        apkbuild_path = pmb.helpers.pmaports.find(args, package)
+        apkbuild = pmb.parse._apkbuild.apkbuild(args,
+                                                f"{apkbuild_path}/APKBUILD")
+
+        if not pmb.helpers.pmaports.check_arches(apkbuild["arch"], arch):
+            print(f"{package}: not enabled for {arch}, skipping")
+            packages.remove(package)
+
+    # No packages: skip build
     if len(packages) == 0:
-        print("no aports changed in this branch")
+        print(f"no packages changed, which can be built for {arch}")
-    else:
+        sys.exit(0)
-        verify_only = common.commit_message_has_string("[ci:skip-build]")
+
-        if verify_only:
+    # Build packages
-            print("WARNING: not building changed packages ([ci:skip-build])!")
+    print(f"building in strict mode for {arch}: {', '.join(packages)}")
-            print("verifying checksums: " + ", ".join(packages))
+    build_strict(packages, arch)
-        else:
-            print("building in strict mode: " + ", ".join(packages))
-        check_build(packages, verify_only)