gitlab-ci: Initial verify-build-push pipeline

This commit is contained in:
Antoine Martin 2023-02-09 17:56:28 -05:00
parent de3d580722
commit 669a372fe7
Signed by: forge
GPG key ID: D62A472A4AA7D541
8 changed files with 653 additions and 89 deletions

View file

@ -1,5 +1,7 @@
stages:
- verify
- build
- deploy
variables:
GIT_STRATEGY: clone
@ -7,112 +9,52 @@ variables:
lint:
stage: verify
image: alpinelinux/apkbuild-lint-tools:latest
interruptible: true
script:
- lint
- |
sudo apk add shellcheck atools doas abuild
export PATH="$PATH:$CI_PROJECT_DIR/.gitlab/bin"
lint
allow_failure: true
only:
- merge_requests
tags:
- docker-alpine
- x86_64
- $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
.build:
stage: verify
image: alpinelinux/alpine-gitlab-ci:latest
build:
stage: build
interruptible: true
script:
- build.sh
- |
sudo apk add alpine-sdk lua-aports doas
doas addgroup $USER abuild
export PATH="$PATH:$CI_PROJECT_DIR/.gitlab/bin"
sudo -Eu $USER build.sh
artifacts:
paths:
- packages/
- keys/
- logs/
expire_in: 1 day
expire_in: 7 days
when: always
only:
- merge_requests
build-x86_64:
extends: .build
artifacts:
name: MR${CI_MERGE_REQUEST_ID}_x86_64
tags:
- docker-alpine
- ci-build
- x86_64
- $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
build-x86:
extends: .build
image:
name: alpinelinux/alpine-gitlab-ci:latest-x86
entrypoint: ["linux32", "sh", "-c"]
artifacts:
name: MR${CI_MERGE_REQUEST_ID}_x86
push:
interruptible: true
stage: deploy
needs:
- job: build
artifacts: true
script:
- |
sudo apk add git abuild
export PATH="$PATH:$CI_PROJECT_DIR/.gitlab/bin"
push.sh
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
when: manual
tags:
- docker-alpine
- ci-build
- x86
build-s390x:
extends: .build
artifacts:
name: MR${CI_MERGE_REQUEST_ID}_s390x
tags:
- docker-alpine
- ci-build
- s390x
build-ppc64le:
extends: .build
artifacts:
name: MR${CI_MERGE_REQUEST_ID}_ppc64le
tags:
- docker-alpine
- ci-build
- ppc64le
build-aarch64:
extends: .build
artifacts:
name: MR${CI_MERGE_REQUEST_ID}_aarch64
tags:
- docker-alpine
- ci-build
- aarch64
build-armv7:
extends: .build
image:
name: alpinelinux/alpine-gitlab-ci:latest-armv7
entrypoint: ["linux32", "sh", "-c"]
artifacts:
name: MR${CI_MERGE_REQUEST_ID}_armv7
tags:
- docker-alpine
- ci-build
- armv7
build-armhf:
extends: .build
image:
name: alpinelinux/alpine-gitlab-ci:latest-armhf
entrypoint: ["linux32", "sh", "-c"]
artifacts:
name: MR${CI_MERGE_REQUEST_ID}_armhf
tags:
- docker-alpine
- ci-build
- armhf
build-riscv64-emulated:
extends: .build
when: manual
artifacts:
name: MR${CI_MERGE_REQUEST_ID}_riscv64
tags:
- docker-alpine
- ci-build
- riscv64
- $CI_MERGE_REQUEST_TARGET_BRANCH_NAME

111
.gitlab/bin/APKBUILD_SHIM Executable file
View file

@ -0,0 +1,111 @@
#!/bin/sh
set -e
arch=
builddir=
checkdepends=
depends=
depends_dev=
depends_doc=
depends_libs=
depends_openrc=
depends_static=
install=
install_if=
langdir=
ldpath=
license=
makedepends=
makedepends_build=
makedepends_host=
md5sums=
options=
patch_args=
pkgbasedir=
pkgdesc=
pkgdir=
pkgname=
pkgrel=
pkgver=
pkggroups=
pkgusers=
provides=
provider_priority=
replaces=
sha256sums=
sha512sums=
sonameprefix=
source=
srcdir=
startdir=
subpackages=
subpkgdir=
subpkgname=
triggers=
url=
# abuild.conf
CFLAGS=
CXXFLAGS=
CPPFLAGS=
LDFLAGS=
JOBS=
MAKEFLAGS=
CMAKE_CROSSOPTS=
. ./APKBUILD
: "$arch"
: "$builddir"
: "$checkdepends"
: "$depends"
: "$depends_dev"
: "$depends_doc"
: "$depends_libs"
: "$depends_openrc"
: "$depends_static"
: "$install"
: "$install_if"
: "$langdir"
: "$ldpath"
: "$license"
: "$makedepends"
: "$makedepends_build"
: "$makedepends_host"
: "$md5sums"
: "$options"
: "$patch_args"
: "$pkgbasedir"
: "$pkgdesc"
: "$pkgdir"
: "$pkgname"
: "$pkgrel"
: "$pkgver"
: "$pkggroups"
: "$pkgusers"
: "$provides"
: "$provider_priority"
: "$replaces"
: "$sha256sums"
: "$sha512sums"
: "$sonameprefix"
: "$source"
: "$srcdir"
: "$startdir"
: "$subpackages"
: "$subpkgdir"
: "$subpkgname"
: "$triggers"
: "$url"
# abuild.conf
: "$CFLAGS"
: "$CXXFLAGS"
: "$CPPFLAGS"
: "$LDFLAGS"
: "$JOBS"
: "$MAKEFLAGS"
: "$CMAKE_CROSSOPTS"

16
.gitlab/bin/apkbuild-shellcheck Executable file
View file

@ -0,0 +1,16 @@
#!/bin/sh
shellcheck -s ash \
-e SC3043 \
-e SC3057 \
-e SC3060 \
-e SC2016 \
-e SC2086 \
-e SC2169 \
-e SC2155 \
-e SC2100 \
-e SC2209 \
-e SC2030 \
-e SC2031 \
-e SC1090 \
-xa $CI_PROJECT_DIR/.gitlab/bin/APKBUILD_SHIM

263
.gitlab/bin/build.sh Executable file
View file

@ -0,0 +1,263 @@
#!/bin/sh
# shellcheck disable=SC3043
. $CI_PROJECT_DIR/.gitlab/bin/functions.sh
# shellcheck disable=SC3040
set -eu -o pipefail
readonly APORTSDIR=$CI_PROJECT_DIR
readonly REPOS="cross backports user"
readonly ALPINE_REPOS="main community testing"
readonly ARCH=$(apk --print-arch)
# gitlab variables
readonly BASEBRANCH=$CI_MERGE_REQUEST_TARGET_BRANCH_NAME
: "${REPODEST:=$HOME/packages}"
: "${MIRROR:=https://lab.ilot.io/ayakael/repo-apk/-/raw}"
: "${ALPINE_MIRROR:=http://dl-cdn.alpinelinux.org/alpine}"
: "${MAX_ARTIFACT_SIZE:=300000000}" #300M
: "${CI_DEBUG_BUILD:=}"
: "${CI_ALPINE_BUILD_OFFSET:=0}"
: "${CI_ALPINE_BUILD_LIMIT:=9999}"
msg() {
local color=${2:-green}
case "$color" in
red) color="31";;
green) color="32";;
yellow) color="33";;
blue) color="34";;
*) color="32";;
esac
printf "\033[1;%sm>>>\033[1;0m %s\n" "$color" "$1" | xargs >&2
}
verbose() {
echo "> " "$@"
# shellcheck disable=SC2068
$@
}
debugging() {
[ -n "$CI_DEBUG_BUILD" ]
}
debug() {
if debugging; then
verbose "$@"
fi
}
die() {
msg "$1" red
exit 1
}
capture_stderr() {
"$@" 2>&1
}
report() {
report=$1
reportsdir=$APORTSDIR/logs/
mkdir -p "$reportsdir"
tee -a "$reportsdir/$report.log"
}
get_release() {
case $BASEBRANCH in
v*) echo v"${BASEBRANCH%-*}";;
master) echo edge;;
*) die "Branch \"$BASEBRANCH\" not supported!"
esac
}
build_aport() {
local repo="$1" aport="$2"
cd "$APORTSDIR/$repo/$aport"
if abuild -r 2>&1 | report "build-$aport"; then
checkapk | report "checkapk-$aport" || true
aport_ok="$aport_ok $repo/$aport"
else
aport_ng="$aport_ng $repo/$aport"
fi
}
check_aport() {
local repo="$1" aport="$2"
cd "$APORTSDIR/$repo/$aport"
if ! abuild check_arch 2>/dev/null; then
aport_na="$aport_na $repo/$aport"
return 1
fi
}
set_repositories_for() {
local target_repo="$1" repos='' repo=''
local release
release=$(get_release)
for repo in $REPOS; do
[ "$release" == "edge" ] && [ "$repo" == "backports" ] && continue
repos="$repos $MIRROR/$release/$repo $REPODEST/$repo"
[ "$repo" = "$target_repo" ] && break
done
doas sh -c "printf '%s\n' $repos >> /etc/apk/repositories"
doas apk update
}
apply_offset_limit() {
start=$1
limit=$2
end=$((start+limit))
sed -n "$((start+1)),${end}p"
}
setup_system() {
local repos='' repo=''
local release
release=$(get_release)
for repo in $ALPINE_REPOS; do
[ "$release" != "edge" ] && [ "$repo" == "testing" ] && continue
repos="$repos $ALPINE_MIRROR/$release/$repo"
done
doas sh -c "printf '%s\n' $repos > /etc/apk/repositories"
doas apk -U upgrade -a || doas apk fix || die "Failed to up/downgrade system"
gitlab_key_to_rsa $ABUILD_KEY PRIVATE $HOME/.abuild/key.rsa
gitlab_key_to_rsa $ABUILD_KEY_PUB PUBLIC $HOME/.abuild/key.rsa.pub
chmod 700 $HOME/.abuild/key.rsa
echo "PACKAGER_PRIVKEY=$HOME/.abuild/key.rsa" >> $HOME/.abuild/abuild.conf
doas cp $HOME/.abuild/key.rsa.pub /etc/apk/keys/key.rsa.pub
doas sed -i -E 's/export JOBS=[0-9]+$/export JOBS=$(nproc)/' /etc/abuild.conf
( . /etc/abuild.conf && echo "Building with $JOBS jobs" )
mkdir -p "$REPODEST"
git config --global init.defaultBranch master
}
sysinfo() {
printf ">>> Host system information (arch: %s, release: %s) <<<\n" "$ARCH" "$(get_release)"
printf "- Number of Cores: %s\n" "$(nproc)"
printf "- Memory: %s Gb\n" "$(awk '/^MemTotal/ {print ($2/1024/1024)}' /proc/meminfo)"
printf "- Free space: %s\n" "$(df -hP / | awk '/\/$/ {print $4}')"
}
copy_artifacts() {
cd "$APORTSDIR"
packages_size="$(du -sk "$REPODEST" | awk '{print $1 * 1024}')"
if [ -z "$packages_size" ]; then
return
fi
echo "Artifact size: $packages_size bytes"
mkdir -p keys/ packages/
if [ "$packages_size" -lt $MAX_ARTIFACT_SIZE ]; then
msg "Copying packages for artifact upload"
cp -ar "$REPODEST"/* packages/ 2>/dev/null
cp ~/.abuild/*.rsa.pub keys/
else
msg "Artifact size $packages_size larger than max ($MAX_ARTIFACT_SIZE), skipping uploading them" yellow
fi
}
section_start setup "Setting up the system" collapse
if debugging; then
set -x
fi
aport_ok=
aport_na=
aport_ng=
failed=
sysinfo || true
setup_system || die "Failed to setup system"
# git no longer allows to execute in repositories owned by different users
doas chown -R $USER: .
fetch_flags="-qn"
debugging && fetch_flags="-v"
git fetch $fetch_flags "$CI_MERGE_REQUEST_PROJECT_URL" \
"+refs/heads/$BASEBRANCH:refs/heads/$BASEBRANCH"
if debugging; then
merge_base=$(git merge-base "$BASEBRANCH" HEAD) || echo "Could not determine merge-base"
echo "Merge base: $merge_base"
git --version
git config -l
[ -n "$merge_base" ] && git tag -f merge-base "$merge_base"
git --no-pager log -200 --oneline --graph --decorate --all
fi
section_end setup
build_start=$CI_ALPINE_BUILD_OFFSET
build_limit=$CI_ALPINE_BUILD_LIMIT
for repo in $(changed_repos); do
set_repositories_for "$repo"
built_aports=0
changed_aports_in_repo=$(changed_aports "$repo")
changed_aports_in_repo_count=$(echo "$changed_aports_in_repo" | wc -l)
changed_aports_to_build=$(echo "$changed_aports_in_repo" | apply_offset_limit "$build_start" "$build_limit")
msg "Changed aports in $repo:"
# shellcheck disable=SC2086 # Splitting is expected here
printf " - %s\n" $changed_aports_to_build
for pkgname in $changed_aports_to_build; do
section_start "build_$pkgname" "Building package $pkgname"
built_aports=$((built_aports+1))
if check_aport "$repo" "$pkgname"; then
build_aport "$repo" "$pkgname"
fi
section_end "build_$pkgname"
done
build_start=$((build_start-(changed_aports_in_repo_count-built_aports)))
build_limit=$((build_limit-built_aports))
if [ $build_limit -le 0 ]; then
msg "Limit reached, breaking"
break
fi
done
section_start artifacts "Handeling artifacts" collapse
copy_artifacts || true
section_end artifacts
section_start summary "Build summary"
echo "### Build summary ###"
for ok in $aport_ok; do
msg "$ok: build succesfully"
done
for na in $aport_na; do
msg "$na: disabled for $ARCH" yellow
done
for ng in $aport_ng; do
msg "$ng: build failed" red
failed=true
done
section_end summary
if [ "$failed" = true ]; then
exit 1
elif [ -z "$aport_ok" ]; then
msg "No packages found to be built." yellow
fi

20
.gitlab/bin/changed-aports Executable file
View file

@ -0,0 +1,20 @@
#!/bin/sh
if [ $# -lt 1 ]; then
echo "Usage: $0 <basebranch>"
exit 1
fi
if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
echo "Fatal: not inside a git repository"
exit 2
fi
basebranch=$1
if ! git rev-parse --verify --quiet $basebranch >/dev/null; then
# The base branch does not eixst, probably due to a shallow clone
git fetch -v $CI_MERGE_REQUEST_PROJECT_URL.git +refs/heads/$basebranch:refs/heads/$basebranch
fi
git --no-pager diff --diff-filter=ACMR --name-only $basebranch...HEAD -- "*/APKBUILD" | xargs -r -n1 dirname

70
.gitlab/bin/functions.sh Executable file
View file

@ -0,0 +1,70 @@
# shellcheck disable=SC3043
:
# shellcheck disable=SC3040
set -eu -o pipefail
changed_repos() {
: "${APORTSDIR?APORTSDIR missing}"
: "${BASEBRANCH?BASEBRANCH missing}"
cd "$APORTSDIR"
for repo in $REPOS; do
git diff --diff-filter=ACMR --exit-code "$BASEBRANCH"...HEAD -- "$repo" >/dev/null \
|| echo "$repo"
done
}
changed_aports() {
: "${APORTSDIR?APORTSDIR missing}"
: "${BASEBRANCH?BASEBRANCH missing}"
cd "$APORTSDIR"
local repo="$1"
local aports
aports=$(git diff --name-only --diff-filter=ACMR --relative="$repo" \
"$BASEBRANCH"...HEAD -- "*/APKBUILD" | xargs -rn1 dirname)
# shellcheck disable=2086
ap builddirs -d "$APORTSDIR/$repo" $aports 2>/dev/null | xargs -rn1 basename
}
section_start() {
name=${1?arg 1 name missing}
header=${2?arg 2 header missing}
collapsed=$2
timestamp=$(date +%s)
options=""
case $collapsed in
yes|on|collapsed|true) options="[collapsed=true]";;
esac
printf "\e[0Ksection_start:%d:%s%s\r\e[0K%s\n" "$timestamp" "$name" "$options" "$header"
}
section_end() {
name=$1
timestamp=$(date +%s)
printf "\e[0Ksection_end:%d:%s\r\e[0K" "$timestamp" "$name"
}
gitlab_key_to_rsa() {
KEY=$1
TYPE=$2
TGT=$3
TGT_DIR=${TGT%/*}
if [ "$TGT" == "$TGT_DIR" ]; then
TGT_DIR="./"
fi
if [ ! -d "$TGT_DIR" ]; then
mkdir -p "$TGT_DIR"
fi
echo "-----BEGIN RSA $TYPE KEY-----" > "$TGT"
echo $1 | sed 's/.\{64\}/&\
/g' >> "$TGT"
echo "-----END RSA $TYPE KEY-----" >> "$TGT"
}

96
.gitlab/bin/lint Executable file
View file

@ -0,0 +1,96 @@
#!/bin/sh
BLUE="\e[34m"
MAGENTA="\e[35m"
RESET="\e[0m"
readonly BASEBRANCH=$CI_MERGE_REQUEST_TARGET_BRANCH_NAME
verbose() {
echo "> " "$@"
# shellcheck disable=SC2068
$@
}
debugging() {
[ -n "$CI_DEBUG_BUILD" ]
}
debug() {
if debugging; then
verbose "$@"
fi
}
# git no longer allows to execute in repositories owned by different users
doas chown -R gitlab-runner: .
fetch_flags="-qn"
debugging && fetch_flags="-v"
git fetch $fetch_flags "$CI_MERGE_REQUEST_PROJECT_URL" \
"+refs/heads/$BASEBRANCH:refs/heads/$BASEBRANCH"
if debugging; then
merge_base=$(git merge-base "$BASEBRANCH" HEAD)
echo "$merge_base"
git --version
git config -l
git tag merge-base "$merge_base" || { echo "Could not determine merge-base"; exit 50; }
git log --oneline --graph --decorate --all
fi
has_problems=0
for PKG in $(changed-aports "$BASEBRANCH"); do
printf "$BLUE==>$RESET Linting $PKG\n"
(
cd "$PKG"
repo=$(basename $(dirname $PKG));
if [ "$repo" = "main" ]; then
export SKIP_AL1=1
export SKIP_AL13=1
fi
printf "\n\n"
printf "$BLUE"
printf '======================================================\n'
printf " parse APKBUILD:\n"
printf '======================================================'
printf "$RESET\n\n"
( . ./APKBUILD ) || has_problems=1
printf "\n\n"
printf "$BLUE"
printf '======================================================\n'
printf " abuild sanitycheck:\n"
printf '======================================================'
printf "$RESET\n\n"
abuild sanitycheck || has_problems=1
printf "\n\n"
printf "$BLUE"
printf '======================================================\n'
printf " apkbuild-shellcheck:\n"
printf '======================================================'
printf "$RESET\n"
apkbuild-shellcheck || has_problems=1
printf "\n\n"
printf "$BLUE"
printf '======================================================\n'
printf " apkbuild-lint:\n"
printf '======================================================'
printf "$RESET\n\n"
apkbuild-lint APKBUILD || has_problems=1
return $has_problems
) || has_problems=1
echo
done
exit $has_problems

46
.gitlab/bin/push.sh Executable file
View file

@ -0,0 +1,46 @@
#!/bin/sh
# shellcheck disable=SC3043
. $CI_PROJECT_DIR/.gitlab/bin/functions.sh
# shellcheck disable=SC3040
set -eu -o pipefail
readonly APORTSDIR=$CI_PROJECT_DIR
readonly REPOS="cross backports user"
readonly BASEBRANCH=$CI_MERGE_REQUEST_TARGET_BRANCH_NAME
export GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
gitlab_key_to_rsa $ABUILD_KEY PRIVATE $HOME/.abuild/key.rsa
gitlab_key_to_rsa $ABUILD_KEY_PUB PUBLIC $HOME/.abuild/key.rsa.pub
gitlab_key_to_rsa $SSH_KEY PRIVATE $HOME/.ssh/id_rsa
chmod 700 "$HOME"/.ssh/id_rsa
chmod 700 "$HOME"/.abuild/key.rsa
echo "PACKAGER_PRIVKEY=$HOME/.abuild/key.rsa" >> $HOME/.abuild/abuild.conf
echo "REPODEST=$CI_PROJECT_DIR/repo-apk" >> $HOME/.abuild/abuild.conf
doas cp $HOME/.abuild/key.rsa.pub /etc/apk/keys/.
git clone git@lab.ilot.io:ayakael/repo-apk -b edge
for i in $(find packages -type f -name "*.apk"); do
cp $i ${i/packages/repo-apk}
done
fetch_flags="-qn"
git fetch $fetch_flags "$CI_MERGE_REQUEST_PROJECT_URL" \
"+refs/heads/$BASEBRANCH:refs/heads/$BASEBRANCH"
for repo in $(changed_repos); do
mkdir -p $repo/DUMMY
echo "pkgname=DUMMY" > $repo/DUMMY/APKBUILD
cd $repo/DUMMY
abuild index
cd "$CI_PROJECT_DIR"
rm -R $repo/DUMMY
done
git -C repo-apk add .
git -C repo-apk commit -m "Update from $CI_MERGE_REQUEST_IID - $CI_MERGE_REQUEST_TITLE"
git -C repo-apk push