From da52dace85c78f64b6d49b65f0c729a80bfb6c7c Mon Sep 17 00:00:00 2001 From: Oliver Smith Date: Sun, 30 Oct 2022 16:48:16 +0100 Subject: [PATCH] CI: adjust .ci/common.py users to pmbootstrap ci (MR 3608) Adjust all CI scripts that make use of .ci/common.py to 'pmbootstrap ci'. Move all scripts that are not direct entry points to running CI scripts to .ci/lib. Comment out the dtb check, as it is failing. Apparently it didn't run properly before. Let's fix this after this CI change is done. --- .ci/apkbuild-lint.sh | 22 ++++++++++++ .ci/build-aarch64.sh | 1 + .ci/build-armhf.sh | 1 + .ci/build-armv7.sh | 1 + .ci/build-riscv64.sh | 1 + .ci/build-x86.sh | 1 + .ci/build-x86_64.sh | 1 + .ci/build.sh | 8 ----- .ci/check-dtb-install-location.sh | 7 ---- .ci/commits.sh | 20 +++++++++++ .ci/grep.sh | 8 +++++ .ci/lib/add_pmbootstrap_to_import_path | 1 + .../apkbuild_linting.py} | 0 .ci/{ => lib}/build_changed_aports.py | 2 +- .ci/lib/build_changed_aports.sh | 19 ++++++++++ .../check_changed_aports_versions.py | 2 +- .ci/{ => lib}/common.py | 2 +- .ci/{run_testcases.sh => pytest.sh} | 19 +++++++--- .gitlab-ci.yml | 35 +++++++++++-------- 19 files changed, 114 insertions(+), 37 deletions(-) create mode 100755 .ci/apkbuild-lint.sh create mode 120000 .ci/build-aarch64.sh create mode 120000 .ci/build-armhf.sh create mode 120000 .ci/build-armv7.sh create mode 120000 .ci/build-riscv64.sh create mode 120000 .ci/build-x86.sh create mode 120000 .ci/build-x86_64.sh delete mode 100755 .ci/build.sh delete mode 100755 .ci/check-dtb-install-location.sh create mode 100755 .ci/commits.sh create mode 120000 .ci/lib/add_pmbootstrap_to_import_path rename .ci/{apkbuild-linting.py => lib/apkbuild_linting.py} (100%) rename .ci/{ => lib}/build_changed_aports.py (98%) create mode 100755 .ci/lib/build_changed_aports.sh rename .ci/{ => lib}/check_changed_aports_versions.py (99%) rename .ci/{ => lib}/common.py (99%) rename .ci/{run_testcases.sh => pytest.sh} (64%) diff --git a/.ci/apkbuild-lint.sh b/.ci/apkbuild-lint.sh new file mode 100755 index 000000000..7c86890cf --- /dev/null +++ b/.ci/apkbuild-lint.sh @@ -0,0 +1,22 @@ +#!/bin/sh -e +# Description: run apkbuild-lint on modified APKBUILDs +# Options: native +# Use 'native' because it requires git commit history. +# https://postmarktos.org/pmb-ci + +if [ "$(id -u)" = 0 ]; then + set -x + wget "https://gitlab.com/postmarketOS/ci-common/-/raw/master/install_pmbootstrap.sh" + sh ./install_pmbootstrap.sh + exec su "${TESTUSER:-pmos}" -c "sh -e $0" +fi + +# Wrap pmbootstrap to use this repository for --aports +pmaports="$(cd $(dirname $0)/..; pwd -P)" +_pmbootstrap="$(command -v pmbootstrap)" +pmbootstrap() { + "$_pmbootstrap" --aports="$pmaports" "$@" +} + +.ci/lib/apkbuild_linting.py + diff --git a/.ci/build-aarch64.sh b/.ci/build-aarch64.sh new file mode 120000 index 000000000..d322e4a4b --- /dev/null +++ b/.ci/build-aarch64.sh @@ -0,0 +1 @@ +lib/build_changed_aports.sh \ No newline at end of file diff --git a/.ci/build-armhf.sh b/.ci/build-armhf.sh new file mode 120000 index 000000000..d322e4a4b --- /dev/null +++ b/.ci/build-armhf.sh @@ -0,0 +1 @@ +lib/build_changed_aports.sh \ No newline at end of file diff --git a/.ci/build-armv7.sh b/.ci/build-armv7.sh new file mode 120000 index 000000000..d322e4a4b --- /dev/null +++ b/.ci/build-armv7.sh @@ -0,0 +1 @@ +lib/build_changed_aports.sh \ No newline at end of file diff --git a/.ci/build-riscv64.sh b/.ci/build-riscv64.sh new file mode 120000 index 000000000..d322e4a4b --- /dev/null +++ b/.ci/build-riscv64.sh @@ -0,0 +1 @@ +lib/build_changed_aports.sh \ No newline at end of file diff --git a/.ci/build-x86.sh b/.ci/build-x86.sh new file mode 120000 index 000000000..d322e4a4b --- /dev/null +++ b/.ci/build-x86.sh @@ -0,0 +1 @@ +lib/build_changed_aports.sh \ No newline at end of file diff --git a/.ci/build-x86_64.sh b/.ci/build-x86_64.sh new file mode 120000 index 000000000..d322e4a4b --- /dev/null +++ b/.ci/build-x86_64.sh @@ -0,0 +1 @@ +lib/build_changed_aports.sh \ No newline at end of file diff --git a/.ci/build.sh b/.ci/build.sh deleted file mode 100755 index 75a44a5e0..000000000 --- a/.ci/build.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -e -# Convenience wrapper for short arch-specific build jobs in .gitlab-ci.yml - -export PYTHONUNBUFFERED=1 -JOB_ARCH="${CI_JOB_NAME#build-}" - -set -x -su pmos -c ".ci/build_changed_aports.py $JOB_ARCH" diff --git a/.ci/check-dtb-install-location.sh b/.ci/check-dtb-install-location.sh deleted file mode 100755 index ba9765753..000000000 --- a/.ci/check-dtb-install-location.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -e - -if grep -r 'INSTALL_DTBS_PATH="$pkgdir"/usr/share/dtb'; then - echo 'Please do not install dtbs to /usr/share/dtb!' - echo 'Unless you have a good reason not to, please put them in /boot/dtbs' - exit 1 -fi diff --git a/.ci/commits.sh b/.ci/commits.sh new file mode 100755 index 000000000..12eb00d2f --- /dev/null +++ b/.ci/commits.sh @@ -0,0 +1,20 @@ +#!/bin/sh -e +# Copyright 2022 Oliver Smith +# SPDX-License-Identifier: GPL-3.0-or-later +# Description: check pkgver/pkgrel bumps, amount of changed pkgs etc +# Options: native +# Use 'native' because it requires git commit history. +# https://postmarktos.org/pmb-ci + +if [ "$(id -u)" = 0 ]; then + set -x + # In .gitlab-ci.yml currently .ci/pytest.sh runs before this and + # already downloads and runs install_pmbootstrap.sh. + if ! [ -e install_pmbootstrap.sh ]; then + wget "https://gitlab.com/postmarketOS/ci-common/-/raw/master/install_pmbootstrap.sh" + sh ./install_pmbootstrap.sh + fi + exec su "${TESTUSER:-pmos}" -c "sh -e $0" +fi + +.ci/lib/check_changed_aports_versions.py diff --git a/.ci/grep.sh b/.ci/grep.sh index 94e95f177..a40e8ab85 100755 --- a/.ci/grep.sh +++ b/.ci/grep.sh @@ -14,3 +14,11 @@ if grep -qr '(CHANGEME!)' *; then grep --color=always -r '(CHANGEME!)' * exit 1 fi + +# DTBs installed to /usr/share/db +# FIXME: doesn't pass currently, fix it and enable it afterwards +# if grep -r 'INSTALL_DTBS_PATH="$pkgdir"/usr/share/dtb'; then +# echo 'Please do not install dtbs to /usr/share/dtb!' +# echo 'Unless you have a good reason not to, please put them in /boot/dtbs' +# exit 1 +# fi diff --git a/.ci/lib/add_pmbootstrap_to_import_path b/.ci/lib/add_pmbootstrap_to_import_path new file mode 120000 index 000000000..a7978d76d --- /dev/null +++ b/.ci/lib/add_pmbootstrap_to_import_path @@ -0,0 +1 @@ +../testcases/add_pmbootstrap_to_import_path \ No newline at end of file diff --git a/.ci/apkbuild-linting.py b/.ci/lib/apkbuild_linting.py similarity index 100% rename from .ci/apkbuild-linting.py rename to .ci/lib/apkbuild_linting.py diff --git a/.ci/build_changed_aports.py b/.ci/lib/build_changed_aports.py similarity index 98% rename from .ci/build_changed_aports.py rename to .ci/lib/build_changed_aports.py index fd55fa434..2e21559c7 100755 --- a/.ci/build_changed_aports.py +++ b/.ci/lib/build_changed_aports.py @@ -7,7 +7,7 @@ import sys import common # pmbootstrap -import testcases.add_pmbootstrap_to_import_path +import add_pmbootstrap_to_import_path import pmb.parse import pmb.parse._apkbuild import pmb.helpers.pmaports diff --git a/.ci/lib/build_changed_aports.sh b/.ci/lib/build_changed_aports.sh new file mode 100755 index 000000000..252af7ffd --- /dev/null +++ b/.ci/lib/build_changed_aports.sh @@ -0,0 +1,19 @@ +#!/bin/sh -e +# Description: build modified packages for this architecture +# Options: native slow +# https://postmarktos.org/pmb-ci + +if [ "$(id -u)" = 0 ]; then + set -x + wget "https://gitlab.com/postmarketOS/ci-common/-/raw/master/install_pmbootstrap.sh" + sh ./install_pmbootstrap.sh + exec su "${TESTUSER:-pmos}" -c "sh -e $0" +fi + +export PYTHONUNBUFFERED=1 + +# Get the architecture from the symlink we are running +arch="$(echo "$0" | cut -d '-' -f 2 | cut -d '.' -f 1)" + +set -x +.ci/lib/build_changed_aports.py "$arch" diff --git a/.ci/check_changed_aports_versions.py b/.ci/lib/check_changed_aports_versions.py similarity index 99% rename from .ci/check_changed_aports_versions.py rename to .ci/lib/check_changed_aports_versions.py index 3e08e370e..4e34cda82 100755 --- a/.ci/check_changed_aports_versions.py +++ b/.ci/lib/check_changed_aports_versions.py @@ -11,7 +11,7 @@ import subprocess import common # pmbootstrap -import testcases.add_pmbootstrap_to_import_path # noqa +import add_pmbootstrap_to_import_path # noqa import pmb.parse import pmb.parse.version import pmb.helpers.logging diff --git a/.ci/common.py b/.ci/lib/common.py similarity index 99% rename from .ci/common.py rename to .ci/lib/common.py index 7d939a1b6..aae9e679b 100755 --- a/.ci/common.py +++ b/.ci/lib/common.py @@ -17,7 +17,7 @@ def get_pmaports_dir(): global cache if "pmaports_dir" in cache: return cache["pmaports_dir"] - ret = os.path.realpath(os.path.join(os.path.dirname(__file__) + "/..")) + ret = os.path.realpath(os.path.join(os.path.dirname(__file__) + "/../..")) cache["pmaports_dir"] = ret return ret diff --git a/.ci/run_testcases.sh b/.ci/pytest.sh similarity index 64% rename from .ci/run_testcases.sh rename to .ci/pytest.sh index fddacf22f..69930832b 100755 --- a/.ci/run_testcases.sh +++ b/.ci/pytest.sh @@ -1,10 +1,21 @@ #!/bin/sh -e -# Copyright 2021 Oliver Smith +# Copyright 2022 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later +# Description: lint with various python tests +# Options: native +# Use 'native' because it requires pmbootstrap. +# https://postmarktos.org/pmb-ci -# Require pmbootstrap -if ! command -v pmbootstrap > /dev/null; then - echo "ERROR: pmbootstrap needs to be installed." +if [ "$(id -u)" = 0 ]; then + set -x + wget "https://gitlab.com/postmarketOS/ci-common/-/raw/master/install_pmbootstrap.sh" + sh ./install_pmbootstrap.sh pytest + exec su "${TESTUSER:-pmos}" -c "sh -e $0" +fi + +# Require pytest to be installed on the host system +if [ -z "$(command -v pytest)" ]; then + echo "ERROR: pytest command not found, make sure it is in your PATH." exit 1 fi diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f51f92880..0be718772 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -70,16 +70,13 @@ editor-config: - .ci/ec.sh # aports checks (generic) -aports-static: +pytest-commits: stage: lint <<: *only-default - before_script: - - wget "https://gitlab.com/postmarketOS/ci-common/-/raw/master/install_pmbootstrap.sh" - - sh ./install_pmbootstrap.sh pytest script: - - su pmos -c ".ci/run_testcases.sh" - - su pmos -c ".ci/check_changed_aports_versions.py" - - su pmos -c ".ci/check-dtb-install-location.sh" + - .ci/lib/gitlab_prepare_ci.sh + - .ci/pytest.sh + - .ci/commits.sh artifacts: when: on_failure paths: @@ -103,11 +100,9 @@ distfile-check: # APKBUILD linting aport-lint: stage: lint - before_script: - - wget "https://gitlab.com/postmarketOS/ci-common/-/raw/master/install_pmbootstrap.sh" - - sh ./install_pmbootstrap.sh script: - - su pmos -c ".ci/apkbuild-linting.py" + - .ci/lib/gitlab_prepare_ci.sh + - .ci/apkbuild-lint.sh only: - merge_requests allow_failure: true @@ -145,10 +140,8 @@ mr-settings: stage: build <<: *only-default before_script: - - wget "https://gitlab.com/postmarketOS/ci-common/-/raw/master/install_pmbootstrap.sh" - - sh ./install_pmbootstrap.sh - script: - - .ci/build.sh + - .ci/lib/gitlab_prepare_ci.sh + after_script: - cp -r /home/pmos/.local/var/pmbootstrap/packages/ packages/ || true artifacts: expire_in: 1 week @@ -158,18 +151,30 @@ mr-settings: build-x86_64: extends: .build + script: + - .ci/build-x86_64.sh build-x86: extends: .build + script: + - .ci/build-x86.sh build-aarch64: extends: .build + script: + - .ci/build-aarch64.sh build-armv7: extends: .build + script: + - .ci/build-armv7.sh build-armhf: extends: .build + script: + - .ci/build-armhf.sh build-riscv64: extends: .build + script: + - .ci/build-riscv64.sh