From 669a372fe7064dd1be4c8f477b56fab9c14f0785 Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Thu, 9 Feb 2023 17:56:28 -0500 Subject: [PATCH] gitlab-ci: Initial verify-build-push pipeline --- .gitlab-ci.yml | 120 ++++----------- .gitlab/bin/APKBUILD_SHIM | 111 ++++++++++++++ .gitlab/bin/apkbuild-shellcheck | 16 ++ .gitlab/bin/build.sh | 263 ++++++++++++++++++++++++++++++++ .gitlab/bin/changed-aports | 20 +++ .gitlab/bin/functions.sh | 70 +++++++++ .gitlab/bin/lint | 96 ++++++++++++ .gitlab/bin/push.sh | 46 ++++++ 8 files changed, 653 insertions(+), 89 deletions(-) create mode 100755 .gitlab/bin/APKBUILD_SHIM create mode 100755 .gitlab/bin/apkbuild-shellcheck create mode 100755 .gitlab/bin/build.sh create mode 100755 .gitlab/bin/changed-aports create mode 100755 .gitlab/bin/functions.sh create mode 100755 .gitlab/bin/lint create mode 100755 .gitlab/bin/push.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b812586..00b93d4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,7 @@ stages: - verify + - build + - deploy variables: GIT_STRATEGY: clone @@ -7,112 +9,52 @@ variables: lint: stage: verify - image: alpinelinux/apkbuild-lint-tools:latest interruptible: true script: - - lint + - | + sudo apk add shellcheck atools doas abuild + export PATH="$PATH:$CI_PROJECT_DIR/.gitlab/bin" + lint allow_failure: true only: - merge_requests tags: - - docker-alpine - - x86_64 + - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME -.build: - stage: verify - image: alpinelinux/alpine-gitlab-ci:latest +build: + stage: build interruptible: true script: - - build.sh + - | + sudo apk add alpine-sdk lua-aports doas + doas addgroup $USER abuild + export PATH="$PATH:$CI_PROJECT_DIR/.gitlab/bin" + sudo -Eu $USER build.sh artifacts: paths: - packages/ - keys/ - logs/ - expire_in: 1 day + expire_in: 7 days when: always only: - merge_requests - -build-x86_64: - extends: .build - artifacts: - name: MR${CI_MERGE_REQUEST_ID}_x86_64 tags: - - docker-alpine - - ci-build - - x86_64 + - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME -build-x86: - extends: .build - image: - name: alpinelinux/alpine-gitlab-ci:latest-x86 - entrypoint: ["linux32", "sh", "-c"] - artifacts: - name: MR${CI_MERGE_REQUEST_ID}_x86 +push: + interruptible: true + stage: deploy + needs: + - job: build + artifacts: true + script: + - | + sudo apk add git abuild + export PATH="$PATH:$CI_PROJECT_DIR/.gitlab/bin" + push.sh + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + when: manual tags: - - docker-alpine - - ci-build - - x86 - -build-s390x: - extends: .build - artifacts: - name: MR${CI_MERGE_REQUEST_ID}_s390x - tags: - - docker-alpine - - ci-build - - s390x - -build-ppc64le: - extends: .build - artifacts: - name: MR${CI_MERGE_REQUEST_ID}_ppc64le - tags: - - docker-alpine - - ci-build - - ppc64le - -build-aarch64: - extends: .build - artifacts: - name: MR${CI_MERGE_REQUEST_ID}_aarch64 - tags: - - docker-alpine - - ci-build - - aarch64 - -build-armv7: - extends: .build - image: - name: alpinelinux/alpine-gitlab-ci:latest-armv7 - entrypoint: ["linux32", "sh", "-c"] - artifacts: - name: MR${CI_MERGE_REQUEST_ID}_armv7 - tags: - - docker-alpine - - ci-build - - armv7 - -build-armhf: - extends: .build - image: - name: alpinelinux/alpine-gitlab-ci:latest-armhf - entrypoint: ["linux32", "sh", "-c"] - artifacts: - name: MR${CI_MERGE_REQUEST_ID}_armhf - tags: - - docker-alpine - - ci-build - - armhf - -build-riscv64-emulated: - extends: .build - when: manual - artifacts: - name: MR${CI_MERGE_REQUEST_ID}_riscv64 - tags: - - docker-alpine - - ci-build - - riscv64 - + - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME diff --git a/.gitlab/bin/APKBUILD_SHIM b/.gitlab/bin/APKBUILD_SHIM new file mode 100755 index 0000000..76577ff --- /dev/null +++ b/.gitlab/bin/APKBUILD_SHIM @@ -0,0 +1,111 @@ +#!/bin/sh + +set -e + +arch= +builddir= +checkdepends= +depends= +depends_dev= +depends_doc= +depends_libs= +depends_openrc= +depends_static= +install= +install_if= +langdir= +ldpath= +license= +makedepends= +makedepends_build= +makedepends_host= +md5sums= +options= +patch_args= +pkgbasedir= +pkgdesc= +pkgdir= +pkgname= +pkgrel= +pkgver= +pkggroups= +pkgusers= +provides= +provider_priority= +replaces= +sha256sums= +sha512sums= +sonameprefix= +source= +srcdir= +startdir= +subpackages= +subpkgdir= +subpkgname= +triggers= +url= + +# abuild.conf + +CFLAGS= +CXXFLAGS= +CPPFLAGS= +LDFLAGS= +JOBS= +MAKEFLAGS= +CMAKE_CROSSOPTS= + +. ./APKBUILD + +: "$arch" +: "$builddir" +: "$checkdepends" +: "$depends" +: "$depends_dev" +: "$depends_doc" +: "$depends_libs" +: "$depends_openrc" +: "$depends_static" +: "$install" +: "$install_if" +: "$langdir" +: "$ldpath" +: "$license" +: "$makedepends" +: "$makedepends_build" +: "$makedepends_host" +: "$md5sums" +: "$options" +: "$patch_args" +: "$pkgbasedir" +: "$pkgdesc" +: "$pkgdir" +: "$pkgname" +: "$pkgrel" +: "$pkgver" +: "$pkggroups" +: "$pkgusers" +: "$provides" +: "$provider_priority" +: "$replaces" +: "$sha256sums" +: "$sha512sums" +: "$sonameprefix" +: "$source" +: "$srcdir" +: "$startdir" +: "$subpackages" +: "$subpkgdir" +: "$subpkgname" +: "$triggers" +: "$url" + +# abuild.conf + +: "$CFLAGS" +: "$CXXFLAGS" +: "$CPPFLAGS" +: "$LDFLAGS" +: "$JOBS" +: "$MAKEFLAGS" +: "$CMAKE_CROSSOPTS" diff --git a/.gitlab/bin/apkbuild-shellcheck b/.gitlab/bin/apkbuild-shellcheck new file mode 100755 index 0000000..3126684 --- /dev/null +++ b/.gitlab/bin/apkbuild-shellcheck @@ -0,0 +1,16 @@ +#!/bin/sh + +shellcheck -s ash \ + -e SC3043 \ + -e SC3057 \ + -e SC3060 \ + -e SC2016 \ + -e SC2086 \ + -e SC2169 \ + -e SC2155 \ + -e SC2100 \ + -e SC2209 \ + -e SC2030 \ + -e SC2031 \ + -e SC1090 \ + -xa $CI_PROJECT_DIR/.gitlab/bin/APKBUILD_SHIM diff --git a/.gitlab/bin/build.sh b/.gitlab/bin/build.sh new file mode 100755 index 0000000..c408171 --- /dev/null +++ b/.gitlab/bin/build.sh @@ -0,0 +1,263 @@ +#!/bin/sh +# shellcheck disable=SC3043 + +. $CI_PROJECT_DIR/.gitlab/bin/functions.sh + +# shellcheck disable=SC3040 +set -eu -o pipefail + +readonly APORTSDIR=$CI_PROJECT_DIR +readonly REPOS="cross backports user" +readonly ALPINE_REPOS="main community testing" +readonly ARCH=$(apk --print-arch) +# gitlab variables +readonly BASEBRANCH=$CI_MERGE_REQUEST_TARGET_BRANCH_NAME + +: "${REPODEST:=$HOME/packages}" +: "${MIRROR:=https://lab.ilot.io/ayakael/repo-apk/-/raw}" +: "${ALPINE_MIRROR:=http://dl-cdn.alpinelinux.org/alpine}" +: "${MAX_ARTIFACT_SIZE:=300000000}" #300M +: "${CI_DEBUG_BUILD:=}" + +: "${CI_ALPINE_BUILD_OFFSET:=0}" +: "${CI_ALPINE_BUILD_LIMIT:=9999}" + +msg() { + local color=${2:-green} + case "$color" in + red) color="31";; + green) color="32";; + yellow) color="33";; + blue) color="34";; + *) color="32";; + esac + printf "\033[1;%sm>>>\033[1;0m %s\n" "$color" "$1" | xargs >&2 +} + +verbose() { + echo "> " "$@" + # shellcheck disable=SC2068 + $@ +} + +debugging() { + [ -n "$CI_DEBUG_BUILD" ] +} + +debug() { + if debugging; then + verbose "$@" + fi +} + +die() { + msg "$1" red + exit 1 +} + +capture_stderr() { + "$@" 2>&1 +} + +report() { + report=$1 + + reportsdir=$APORTSDIR/logs/ + mkdir -p "$reportsdir" + + tee -a "$reportsdir/$report.log" +} + +get_release() { + case $BASEBRANCH in + v*) echo v"${BASEBRANCH%-*}";; + master) echo edge;; + *) die "Branch \"$BASEBRANCH\" not supported!" + esac +} + +build_aport() { + local repo="$1" aport="$2" + cd "$APORTSDIR/$repo/$aport" + if abuild -r 2>&1 | report "build-$aport"; then + checkapk | report "checkapk-$aport" || true + aport_ok="$aport_ok $repo/$aport" + else + aport_ng="$aport_ng $repo/$aport" + fi +} + +check_aport() { + local repo="$1" aport="$2" + cd "$APORTSDIR/$repo/$aport" + if ! abuild check_arch 2>/dev/null; then + aport_na="$aport_na $repo/$aport" + return 1 + fi +} + +set_repositories_for() { + local target_repo="$1" repos='' repo='' + local release + + release=$(get_release) + for repo in $REPOS; do + [ "$release" == "edge" ] && [ "$repo" == "backports" ] && continue + repos="$repos $MIRROR/$release/$repo $REPODEST/$repo" + [ "$repo" = "$target_repo" ] && break + done + doas sh -c "printf '%s\n' $repos >> /etc/apk/repositories" + doas apk update +} + +apply_offset_limit() { + start=$1 + limit=$2 + end=$((start+limit)) + + sed -n "$((start+1)),${end}p" +} + +setup_system() { + local repos='' repo='' + local release + + release=$(get_release) + for repo in $ALPINE_REPOS; do + [ "$release" != "edge" ] && [ "$repo" == "testing" ] && continue + repos="$repos $ALPINE_MIRROR/$release/$repo" + done + doas sh -c "printf '%s\n' $repos > /etc/apk/repositories" + doas apk -U upgrade -a || doas apk fix || die "Failed to up/downgrade system" + gitlab_key_to_rsa $ABUILD_KEY PRIVATE $HOME/.abuild/key.rsa + gitlab_key_to_rsa $ABUILD_KEY_PUB PUBLIC $HOME/.abuild/key.rsa.pub + chmod 700 $HOME/.abuild/key.rsa + echo "PACKAGER_PRIVKEY=$HOME/.abuild/key.rsa" >> $HOME/.abuild/abuild.conf + doas cp $HOME/.abuild/key.rsa.pub /etc/apk/keys/key.rsa.pub + + doas sed -i -E 's/export JOBS=[0-9]+$/export JOBS=$(nproc)/' /etc/abuild.conf + ( . /etc/abuild.conf && echo "Building with $JOBS jobs" ) + mkdir -p "$REPODEST" + git config --global init.defaultBranch master +} + +sysinfo() { + printf ">>> Host system information (arch: %s, release: %s) <<<\n" "$ARCH" "$(get_release)" + printf "- Number of Cores: %s\n" "$(nproc)" + printf "- Memory: %s Gb\n" "$(awk '/^MemTotal/ {print ($2/1024/1024)}' /proc/meminfo)" + printf "- Free space: %s\n" "$(df -hP / | awk '/\/$/ {print $4}')" +} + +copy_artifacts() { + cd "$APORTSDIR" + + packages_size="$(du -sk "$REPODEST" | awk '{print $1 * 1024}')" + if [ -z "$packages_size" ]; then + return + fi + + echo "Artifact size: $packages_size bytes" + + mkdir -p keys/ packages/ + + if [ "$packages_size" -lt $MAX_ARTIFACT_SIZE ]; then + msg "Copying packages for artifact upload" + cp -ar "$REPODEST"/* packages/ 2>/dev/null + cp ~/.abuild/*.rsa.pub keys/ + else + msg "Artifact size $packages_size larger than max ($MAX_ARTIFACT_SIZE), skipping uploading them" yellow + fi +} + +section_start setup "Setting up the system" collapse + +if debugging; then + set -x +fi + +aport_ok= +aport_na= +aport_ng= +failed= + +sysinfo || true +setup_system || die "Failed to setup system" + +# git no longer allows to execute in repositories owned by different users +doas chown -R $USER: . + +fetch_flags="-qn" +debugging && fetch_flags="-v" + +git fetch $fetch_flags "$CI_MERGE_REQUEST_PROJECT_URL" \ + "+refs/heads/$BASEBRANCH:refs/heads/$BASEBRANCH" + +if debugging; then + merge_base=$(git merge-base "$BASEBRANCH" HEAD) || echo "Could not determine merge-base" + echo "Merge base: $merge_base" + git --version + git config -l + [ -n "$merge_base" ] && git tag -f merge-base "$merge_base" + git --no-pager log -200 --oneline --graph --decorate --all +fi + +section_end setup + +build_start=$CI_ALPINE_BUILD_OFFSET +build_limit=$CI_ALPINE_BUILD_LIMIT + +for repo in $(changed_repos); do + set_repositories_for "$repo" + built_aports=0 + changed_aports_in_repo=$(changed_aports "$repo") + changed_aports_in_repo_count=$(echo "$changed_aports_in_repo" | wc -l) + changed_aports_to_build=$(echo "$changed_aports_in_repo" | apply_offset_limit "$build_start" "$build_limit") + + msg "Changed aports in $repo:" + # shellcheck disable=SC2086 # Splitting is expected here + printf " - %s\n" $changed_aports_to_build + for pkgname in $changed_aports_to_build; do + section_start "build_$pkgname" "Building package $pkgname" + built_aports=$((built_aports+1)) + if check_aport "$repo" "$pkgname"; then + build_aport "$repo" "$pkgname" + fi + section_end "build_$pkgname" + done + + build_start=$((build_start-(changed_aports_in_repo_count-built_aports))) + build_limit=$((build_limit-built_aports)) + + if [ $build_limit -le 0 ]; then + msg "Limit reached, breaking" + break + fi +done + +section_start artifacts "Handeling artifacts" collapse +copy_artifacts || true +section_end artifacts + +section_start summary "Build summary" + +echo "### Build summary ###" + +for ok in $aport_ok; do + msg "$ok: build succesfully" +done + +for na in $aport_na; do + msg "$na: disabled for $ARCH" yellow +done + +for ng in $aport_ng; do + msg "$ng: build failed" red + failed=true +done +section_end summary + +if [ "$failed" = true ]; then + exit 1 +elif [ -z "$aport_ok" ]; then + msg "No packages found to be built." yellow +fi diff --git a/.gitlab/bin/changed-aports b/.gitlab/bin/changed-aports new file mode 100755 index 0000000..4541230 --- /dev/null +++ b/.gitlab/bin/changed-aports @@ -0,0 +1,20 @@ +#!/bin/sh + +if [ $# -lt 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then + echo "Fatal: not inside a git repository" + exit 2 +fi + +basebranch=$1 + +if ! git rev-parse --verify --quiet $basebranch >/dev/null; then + # The base branch does not eixst, probably due to a shallow clone + git fetch -v $CI_MERGE_REQUEST_PROJECT_URL.git +refs/heads/$basebranch:refs/heads/$basebranch +fi + +git --no-pager diff --diff-filter=ACMR --name-only $basebranch...HEAD -- "*/APKBUILD" | xargs -r -n1 dirname diff --git a/.gitlab/bin/functions.sh b/.gitlab/bin/functions.sh new file mode 100755 index 0000000..4e1f04a --- /dev/null +++ b/.gitlab/bin/functions.sh @@ -0,0 +1,70 @@ +# shellcheck disable=SC3043 + +: + +# shellcheck disable=SC3040 +set -eu -o pipefail + +changed_repos() { + : "${APORTSDIR?APORTSDIR missing}" + : "${BASEBRANCH?BASEBRANCH missing}" + + cd "$APORTSDIR" + for repo in $REPOS; do + git diff --diff-filter=ACMR --exit-code "$BASEBRANCH"...HEAD -- "$repo" >/dev/null \ + || echo "$repo" + done +} + +changed_aports() { + : "${APORTSDIR?APORTSDIR missing}" + : "${BASEBRANCH?BASEBRANCH missing}" + + cd "$APORTSDIR" + local repo="$1" + local aports + + aports=$(git diff --name-only --diff-filter=ACMR --relative="$repo" \ + "$BASEBRANCH"...HEAD -- "*/APKBUILD" | xargs -rn1 dirname) + + # shellcheck disable=2086 + ap builddirs -d "$APORTSDIR/$repo" $aports 2>/dev/null | xargs -rn1 basename +} + +section_start() { + name=${1?arg 1 name missing} + header=${2?arg 2 header missing} + collapsed=$2 + timestamp=$(date +%s) + + options="" + case $collapsed in + yes|on|collapsed|true) options="[collapsed=true]";; + esac + + printf "\e[0Ksection_start:%d:%s%s\r\e[0K%s\n" "$timestamp" "$name" "$options" "$header" +} + +section_end() { + name=$1 + timestamp=$(date +%s) + + printf "\e[0Ksection_end:%d:%s\r\e[0K" "$timestamp" "$name" +} + +gitlab_key_to_rsa() { + KEY=$1 + TYPE=$2 + TGT=$3 + TGT_DIR=${TGT%/*} + if [ "$TGT" == "$TGT_DIR" ]; then + TGT_DIR="./" + fi + if [ ! -d "$TGT_DIR" ]; then + mkdir -p "$TGT_DIR" + fi + echo "-----BEGIN RSA $TYPE KEY-----" > "$TGT" + echo $1 | sed 's/.\{64\}/&\ +/g' >> "$TGT" + echo "-----END RSA $TYPE KEY-----" >> "$TGT" +} diff --git a/.gitlab/bin/lint b/.gitlab/bin/lint new file mode 100755 index 0000000..bd11c3a --- /dev/null +++ b/.gitlab/bin/lint @@ -0,0 +1,96 @@ +#!/bin/sh + +BLUE="\e[34m" +MAGENTA="\e[35m" +RESET="\e[0m" + +readonly BASEBRANCH=$CI_MERGE_REQUEST_TARGET_BRANCH_NAME + +verbose() { + echo "> " "$@" + # shellcheck disable=SC2068 + $@ +} + +debugging() { + [ -n "$CI_DEBUG_BUILD" ] +} + +debug() { + if debugging; then + verbose "$@" + fi +} + +# git no longer allows to execute in repositories owned by different users +doas chown -R gitlab-runner: . + +fetch_flags="-qn" +debugging && fetch_flags="-v" + +git fetch $fetch_flags "$CI_MERGE_REQUEST_PROJECT_URL" \ + "+refs/heads/$BASEBRANCH:refs/heads/$BASEBRANCH" + +if debugging; then + merge_base=$(git merge-base "$BASEBRANCH" HEAD) + echo "$merge_base" + git --version + git config -l + git tag merge-base "$merge_base" || { echo "Could not determine merge-base"; exit 50; } + git log --oneline --graph --decorate --all +fi + +has_problems=0 + +for PKG in $(changed-aports "$BASEBRANCH"); do + printf "$BLUE==>$RESET Linting $PKG\n" + + ( + cd "$PKG" + + repo=$(basename $(dirname $PKG)); + + if [ "$repo" = "main" ]; then + export SKIP_AL1=1 + export SKIP_AL13=1 + fi + + printf "\n\n" + printf "$BLUE" + printf '======================================================\n' + printf " parse APKBUILD:\n" + printf '======================================================' + printf "$RESET\n\n" + ( . ./APKBUILD ) || has_problems=1 + + printf "\n\n" + printf "$BLUE" + printf '======================================================\n' + printf " abuild sanitycheck:\n" + printf '======================================================' + printf "$RESET\n\n" + abuild sanitycheck || has_problems=1 + + printf "\n\n" + printf "$BLUE" + printf '======================================================\n' + printf " apkbuild-shellcheck:\n" + printf '======================================================' + printf "$RESET\n" + apkbuild-shellcheck || has_problems=1 + + printf "\n\n" + printf "$BLUE" + printf '======================================================\n' + printf " apkbuild-lint:\n" + printf '======================================================' + printf "$RESET\n\n" + apkbuild-lint APKBUILD || has_problems=1 + + return $has_problems + ) || has_problems=1 + + echo +done + +exit $has_problems diff --git a/.gitlab/bin/push.sh b/.gitlab/bin/push.sh new file mode 100755 index 0000000..a58cee3 --- /dev/null +++ b/.gitlab/bin/push.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +# shellcheck disable=SC3043 + +. $CI_PROJECT_DIR/.gitlab/bin/functions.sh + +# shellcheck disable=SC3040 +set -eu -o pipefail + +readonly APORTSDIR=$CI_PROJECT_DIR +readonly REPOS="cross backports user" +readonly BASEBRANCH=$CI_MERGE_REQUEST_TARGET_BRANCH_NAME + +export GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" + +gitlab_key_to_rsa $ABUILD_KEY PRIVATE $HOME/.abuild/key.rsa +gitlab_key_to_rsa $ABUILD_KEY_PUB PUBLIC $HOME/.abuild/key.rsa.pub +gitlab_key_to_rsa $SSH_KEY PRIVATE $HOME/.ssh/id_rsa +chmod 700 "$HOME"/.ssh/id_rsa +chmod 700 "$HOME"/.abuild/key.rsa + +echo "PACKAGER_PRIVKEY=$HOME/.abuild/key.rsa" >> $HOME/.abuild/abuild.conf +echo "REPODEST=$CI_PROJECT_DIR/repo-apk" >> $HOME/.abuild/abuild.conf +doas cp $HOME/.abuild/key.rsa.pub /etc/apk/keys/. + +git clone git@lab.ilot.io:ayakael/repo-apk -b edge +for i in $(find packages -type f -name "*.apk"); do + cp $i ${i/packages/repo-apk} +done + +fetch_flags="-qn" +git fetch $fetch_flags "$CI_MERGE_REQUEST_PROJECT_URL" \ + "+refs/heads/$BASEBRANCH:refs/heads/$BASEBRANCH" + +for repo in $(changed_repos); do + mkdir -p $repo/DUMMY + echo "pkgname=DUMMY" > $repo/DUMMY/APKBUILD + cd $repo/DUMMY + abuild index + cd "$CI_PROJECT_DIR" + rm -R $repo/DUMMY +done + +git -C repo-apk add . +git -C repo-apk commit -m "Update from $CI_MERGE_REQUEST_IID - $CI_MERGE_REQUEST_TITLE" +git -C repo-apk push