diff --git a/.forgejo/bin/build.sh b/.forgejo/bin/build.sh new file mode 100755 index 0000000..d9e327f --- /dev/null +++ b/.forgejo/bin/build.sh @@ -0,0 +1,260 @@ +#!/bin/sh +# shellcheck disable=SC3043 + +. /usr/local/lib/functions.sh + +# shellcheck disable=SC3040 +set -eu -o pipefail + +readonly APORTSDIR=$CI_PROJECT_DIR +readonly REPOS="backports user" +readonly ALPINE_REPOS="main community testing" +readonly ARCH=$(apk --print-arch) +# gitlab variables +readonly BASEBRANCH=$CI_MERGE_REQUEST_TARGET_BRANCH_NAME + +: "${REPODEST:=$HOME/packages}" +: "${MIRROR:=https://ayakael.net/api/packages/forge/alpine}" +: "${ALPINE_MIRROR:=http://dl-cdn.alpinelinux.org/alpine}" +: "${MAX_ARTIFACT_SIZE:=300000000}" #300M +: "${CI_DEBUG_BUILD:=}" + +: "${CI_ALPINE_BUILD_OFFSET:=0}" +: "${CI_ALPINE_BUILD_LIMIT:=9999}" + +msg() { + local color=${2:-green} + case "$color" in + red) color="31";; + green) color="32";; + yellow) color="33";; + blue) color="34";; + *) color="32";; + esac + printf "\033[1;%sm>>>\033[1;0m %s\n" "$color" "$1" | xargs >&2 +} + +verbose() { + echo "> " "$@" + # shellcheck disable=SC2068 + $@ +} + +debugging() { + [ -n "$CI_DEBUG_BUILD" ] +} + +debug() { + if debugging; then + verbose "$@" + fi +} + +die() { + msg "$1" red + exit 1 +} + +capture_stderr() { + "$@" 2>&1 +} + +report() { + report=$1 + + reportsdir=$APORTSDIR/logs/ + mkdir -p "$reportsdir" + + tee -a "$reportsdir/$report.log" +} + +get_release() { + case $BASEBRANCH in + v*) echo "$BASEBRANCH";; + edge) echo edge;; + *) die "Branch \"$BASEBRANCH\" not supported!" + esac +} + +build_aport() { + local repo="$1" aport="$2" + cd "$APORTSDIR/$repo/$aport" + if abuild -r 2>&1 | report "build-$aport"; then + checkapk 2>&1 | report "checkapk-$aport" || true + aport_ok="$aport_ok $repo/$aport" + else + aport_ng="$aport_ng $repo/$aport" + fi +} + +check_aport() { + local repo="$1" aport="$2" + cd "$APORTSDIR/$repo/$aport" + if ! abuild check_arch 2>/dev/null; then + aport_na="$aport_na $repo/$aport" + return 1 + fi +} + +set_repositories_for() { + local target_repo="$1" repos='' repo='' + local release + + release=$(get_release) + for repo in $REPOS; do + [ "$repo" = "non-free" ] && continue + [ "$release" == "edge" ] && [ "$repo" == "backports" ] && continue + repos="$repos $MIRROR/$release/$repo $REPODEST/$repo" + [ "$repo" = "$target_repo" ] && break + done + doas sh -c "printf '%s\n' $repos >> /etc/apk/repositories" + doas apk update || true +} + +apply_offset_limit() { + start=$1 + limit=$2 + end=$((start+limit)) + + sed -n "$((start+1)),${end}p" +} + +setup_system() { + local repos='' repo='' + local release + + release=$(get_release) + for repo in $ALPINE_REPOS; do + [ "$release" != "edge" ] && [ "$repo" == "testing" ] && continue + repos="$repos $ALPINE_MIRROR/$release/$repo" + done + doas sh -c "printf '%s\n' $repos > /etc/apk/repositories" + doas apk -U upgrade -a || apk fix || die "Failed to up/downgrade system" + abuild-keygen -ain + doas sed -i -E 's/export JOBS=[0-9]+$/export JOBS=$(nproc)/' /etc/abuild.conf + ( . /etc/abuild.conf && echo "Building with $JOBS jobs" ) + mkdir -p "$REPODEST" + git config --global init.defaultBranch master +} + +sysinfo() { + printf ">>> Host system information (arch: %s, release: %s) <<<\n" "$ARCH" "$(get_release)" + printf "- Number of Cores: %s\n" "$(nproc)" + printf "- Memory: %s Gb\n" "$(awk '/^MemTotal/ {print ($2/1024/1024)}' /proc/meminfo)" + printf "- Free space: %s\n" "$(df -hP / | awk '/\/$/ {print $4}')" +} + +copy_artifacts() { + cd "$APORTSDIR" + + packages_size="$(du -sk "$REPODEST" | awk '{print $1 * 1024}')" + if [ -z "$packages_size" ]; then + return + fi + + echo "Artifact size: $packages_size bytes" + + mkdir -p keys/ packages/ + + if [ "$packages_size" -lt $MAX_ARTIFACT_SIZE ]; then + msg "Copying packages for artifact upload" + cp -ar "$REPODEST"/* packages/ 2>/dev/null + cp ~/.abuild/*.rsa.pub keys/ + else + msg "Artifact size $packages_size larger than max ($MAX_ARTIFACT_SIZE), skipping uploading them" yellow + fi +} + +section_start setup "Setting up the system" collapse + +if debugging; then + set -x +fi + +aport_ok= +aport_na= +aport_ng= +failed= + +sysinfo || true +setup_system || die "Failed to setup system" + +# git no longer allows to execute in repositories owned by different users +doas chown -R buildozer: . + +fetch_flags="-qn" +debugging && fetch_flags="-v" + +git fetch $fetch_flags "$CI_MERGE_REQUEST_PROJECT_URL" \ + "+refs/heads/$BASEBRANCH:refs/heads/$BASEBRANCH" + +if debugging; then + merge_base=$(git merge-base "$BASEBRANCH" HEAD) || echo "Could not determine merge-base" + echo "Merge base: $merge_base" + git --version + git config -l + [ -n "$merge_base" ] && git tag -f merge-base "$merge_base" + git --no-pager log -200 --oneline --graph --decorate --all +fi + +section_end setup + +build_start=$CI_ALPINE_BUILD_OFFSET +build_limit=$CI_ALPINE_BUILD_LIMIT + +for repo in $(changed_repos); do + set_repositories_for "$repo" + built_aports=0 + changed_aports_in_repo=$(changed_aports "$repo") + changed_aports_in_repo_count=$(echo "$changed_aports_in_repo" | wc -l) + changed_aports_to_build=$(echo "$changed_aports_in_repo" | apply_offset_limit "$build_start" "$build_limit") + + msg "Changed aports in $repo:" + # shellcheck disable=SC2086 # Splitting is expected here + printf " - %s\n" $changed_aports_to_build + for pkgname in $changed_aports_to_build; do + section_start "build_$pkgname" "Building package $pkgname" + built_aports=$((built_aports+1)) + if check_aport "$repo" "$pkgname"; then + build_aport "$repo" "$pkgname" + fi + section_end "build_$pkgname" + done + + build_start=$((build_start-(changed_aports_in_repo_count-built_aports))) + build_limit=$((build_limit-built_aports)) + + if [ $build_limit -le 0 ]; then + msg "Limit reached, breaking" + break + fi +done + +section_start artifacts "Handeling artifacts" collapse +copy_artifacts || true +section_end artifacts + +section_start summary "Build summary" + +echo "### Build summary ###" + +for ok in $aport_ok; do + msg "$ok: build succesfully" +done + +for na in $aport_na; do + msg "$na: disabled for $ARCH" yellow +done + +for ng in $aport_ng; do + msg "$ng: build failed" red + failed=true +done +section_end summary + +if [ "$failed" = true ]; then + exit 1 +elif [ -z "$aport_ok" ]; then + msg "No packages found to be built." yellow +fi + diff --git a/.forgejo/bin/check_ver.sh b/.forgejo/bin/check_ver.sh new file mode 100755 index 0000000..d2720f3 --- /dev/null +++ b/.forgejo/bin/check_ver.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# expects the following env variables: +# downstream: downstream repo + +repo=${downstream/*\/} + +curl --silent $downstream/x86_64/APKINDEX.tar.gz | tar -O -zx APKINDEX > APKINDEX + +if [ "$ALL_PACKAGES" == "true" ]; then + owned_by_you=$(awk -F ':' '{if($1=="o"){print $2}}' APKINDEX | sort | uniq) + echo "Found $(printf '%s\n' $owned_by_you | wc -l ) packages" +else + owned_by_you=$(awk -v RS= -v ORS="\n\n" '/m:Antoine Martin \(ayakael\) /' APKINDEX | awk -F ':' '{if($1=="o"){print $2}}' | sort | uniq) + echo "Found $(printf '%s\n' $owned_by_you | wc -l ) packages owned by you" +fi + +rm -f out_of_date not_in_anitya + +for pkg in $owned_by_you; do + upstream_version=$(curl --fail -X GET -sS -H 'Content-Type: application/json' "https://release-monitoring.org/api/v2/packages/?name=$pkg&distribution=Alpine" | jq -r '.items.[].stable_version') + downstream_version=$(sed -n "/^P:$pkg$/,/^$/p" APKINDEX | awk -F ':' '{if($1=="V"){print $2}}' | sort -V | tail -n 1) + downstream_version=${downstream_version/-*} + + # special cases + case $pkg in + freetube) upstream_version=$(curl --fail -X GET -sS -H 'Content-Type: application/json' "https://release-monitoring.org/api/v2/packages/?name=$pkg&distribution=Alpine" | jq -r '.items.[].version' | sed "s|-beta||");; + dotnet9-sdk|dotnet9-stage0) upstream_version=${upstream_version/-*};; + esac + + if [ -z "$upstream_version" ]; then + echo "$pkg not in anitya" + echo "$pkg" >> not_in_anitya + elif [ "$downstream_version" != "$(printf '%s\n' $upstream_version $downstream_version | sort -V | head -n 1)" ]; then + echo "$pkg higher downstream" + continue + elif [ "$upstream_version" != "$downstream_version" ]; then + echo "$pkg upstream version $upstream_version does not match downstream version $downstream_version" + echo "$pkg $downstream_version $upstream_version $repo" >> out_of_date + fi +done diff --git a/.forgejo/bin/clear-repo.sh b/.forgejo/bin/clear-repo.sh new file mode 100755 index 0000000..14110ea --- /dev/null +++ b/.forgejo/bin/clear-repo.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +TARGET_REPO=$1 +ARCH=$2 + + +curl --silent $TARGET_REPO/$ARCH/APKINDEX.tar.gz | tar -O -zx APKINDEX > APKINDEX + +pkgs=$(awk -F ':' '{if($1=="o"){print $2}}' APKINDEX | sort | uniq) + +for pkg in $pkgs; do + pkgvers=$(sed -n "/^P:$pkg$/,/^$/p" APKINDEX | awk -F ':' '{if($1=="V"){print $2}}') + for pkgver in $pkgvers; do + echo "Deleting $pkg-$pkgver of arch $ARCH from $TARGET_REPO" + curl -s --user $FORGE_REPO_USER:$FORGE_REPO_TOKEN -X DELETE $TARGET_REPO/$ARCH/$pkg-$pkgver.apk + done +done diff --git a/.forgejo/bin/create_issue.sh b/.forgejo/bin/create_issue.sh new file mode 100755 index 0000000..d162758 --- /dev/null +++ b/.forgejo/bin/create_issue.sh @@ -0,0 +1,165 @@ +#!/bin/bash + +# expects: +# env variable FORGEJO_TOKEN +# file out_of_date + +IFS=' +' +repo=${downstream/*\/} + +does_it_exist() { + name=$1 + downstream_version=$2 + upstream_version=$3 + repo=$4 + + query="$repo/$name: upgrade to $upstream_version" + query="$(echo $query | sed 's| |%20|g' | sed 's|:|%3A|g' | sed 's|/|%2F|g' )" + + result="$(curl --silent -X 'GET' \ + "$GITHUB_SERVER_URL/api/v1/repos/$GITHUB_REPOSITORY/issues?state=open&q=$query&type=issues" \ + -H 'accept: application/json' \ + -H "authorization: Basic $FORGEJO_TOKEN" + )" + + if [ "$result" == "[]" ]; then + return 1 + fi +} + +is_it_old() { + name=$1 + downstream_version=$2 + upstream_version=$3 + repo=$4 + + query="$repo/$name: upgrade to" + query="$(echo $query | sed 's| |%20|g' | sed 's|:|%3A|g' | sed 's|/|%2F|g' )" + + result="$(curl --silent -X 'GET' \ + "$GITHUB_SERVER_URL/api/v1/repos/$GITHUB_REPOSITORY/issues?state=open&q=$query&type=issues" \ + -H 'accept: application/json' \ + -H "authorization: Basic $FORGEJO_TOKEN" + )" + + result_title="$(echo $result | jq -r '.[].title' )" + result_id="$(echo $result | jq -r '.[].number' )" + result_upstream_version="$(echo $result_title | awk '{print $4}')" + + if [ "$upstream_version" != "$result_upstream_version" ]; then + echo $result_id + else + echo 0 + fi +} + +update_title() { + name=$1 + downstream_version=$2 + upstream_version=$3 + repo=$4 + id=$5 + + result=$(curl --silent -X 'PATCH' \ + "$GITHUB_SERVER_URL/api/v1/repos/$GITHUB_REPOSITORY/issues/$id" \ + -H 'accept: application/json' \ + -H "authorization: Basic $FORGEJO_TOKEN" \ + -H 'Content-Type: application/json' \ + -d "{ + \"title\": \"$repo/$name: upgrade to $upstream_version\" + }" + ) + + return 0 +} + +create_issue() { + name=$1 + downstream_version=$2 + upstream_version=$3 + repo=$4 + + result=$(curl --silent -X 'POST' \ + "$GITHUB_SERVER_URL/api/v1/repos/$GITHUB_REPOSITORY/issues" \ + -H 'accept: application/json' \ + -H "authorization: Basic $FORGEJO_TOKEN" \ + -H 'Content-Type: application/json' \ + -d "{ + \"title\": \"$repo/$name: upgrade to $upstream_version\", + \"labels\": [ + $LABEL_NUMBER + ] + }") + + return 0 +} + +if [ -f out_of_date ]; then + out_of_date="$(cat out_of_date)" + + echo "Detected $(wc -l out_of_date) out-of-date packages, creating issues" + + for pkg in $out_of_date; do + name="$(echo $pkg | awk '{print $1}')" + downstream_version="$(echo $pkg | awk '{print $2}')" + upstream_version="$(echo $pkg | awk '{print $3}')" + repo="$(echo $pkg | awk '{print $4}')" + + if does_it_exist $name $downstream_version $upstream_version $repo; then + echo "Issue for $repo/$name already exists" + continue + fi + + id=$(is_it_old $name $downstream_version $upstream_version $repo) + + if [ "$id" != "0" ] && [ -n "$id" ]; then + echo "Issue for $repo/$name needs updating" + update_title $name $downstream_version $upstream_version $repo $id + continue + fi + + echo "Creating issue for $repo/$name" + create_issue $name $downstream_version $upstream_version $repo + done +fi + +if [ -f not_in_anitya ]; then + query="Add missing $repo packages to anitya" + query="$(echo $query | sed 's| |%20|g')" + + result="$(curl --silent -X 'GET' \ + "$GITHUB_SERVER_URL/api/v1/repos/$GITHUB_REPOSITORY/issues?state=open&q=$query&type=issues" \ + -H 'accept: application/json' \ + -H "authorization: Basic $FORGEJO_TOKEN" + )" + + if [ "$result" == "[]" ]; then + echo "Creating anitya issue" + result=$(curl --silent -X 'POST' \ + "$GITHUB_SERVER_URL/api/v1/repos/$GITHUB_REPOSITORY/issues" \ + -H 'accept: application/json' \ + -H "authorization: Basic $FORGEJO_TOKEN" \ + -H 'Content-Type: application/json' \ + -d "{ + \"title\": \"Add missing $repo packages to anitya\", + \"body\": \"- [ ] $(sed '{:q;N;s/\n/\\n- [ ] /g;t q}' not_in_anitya)\", + \"labels\": [ + $LABEL_NUMBER + ] + }") + + else + echo "Updating anitya issue" + result_id="$(echo $result | jq -r '.[].number' )" + result=$(curl --silent -X 'PATCH' \ + "$GITHUB_SERVER_URL/api/v1/repos/$GITHUB_REPOSITORY/issues/$result_id" \ + -H 'accept: application/json' \ + -H "authorization: Basic $FORGEJO_TOKEN" \ + -H 'Content-Type: application/json' \ + -d "{ + \"body\": \"- [ ] $(sed '{:q;N;s/\n/\\n- [ ] /g;t q}' not_in_anitya)\" + }" + ) + fi +fi diff --git a/.forgejo/workflows/build-aarch64.yaml b/.forgejo/workflows/build-aarch64.yaml index d6738b2..59f90c9 100644 --- a/.forgejo/workflows/build-aarch64.yaml +++ b/.forgejo/workflows/build-aarch64.yaml @@ -24,8 +24,8 @@ jobs: fetch-depth: 500 - name: Package build run: | - doas patch -d / -p1 -i ${{ github.workspace }}/.forgejo/patches/build.patch - build.sh + ${{ github.workspace }}/.forgejo/bin/build.sh + touch packages/dummy - name: Package upload uses: forgejo/upload-artifact@v3 with: diff --git a/.forgejo/workflows/build-x86_64.yaml b/.forgejo/workflows/build-x86_64.yaml index 8731799..298a213 100644 --- a/.forgejo/workflows/build-x86_64.yaml +++ b/.forgejo/workflows/build-x86_64.yaml @@ -24,8 +24,8 @@ jobs: fetch-depth: 500 - name: Package build run: | - doas patch -d / -p1 -i ${{ github.workspace }}/.forgejo/patches/build.patch - build.sh + ${{ github.workspace }}/.forgejo/bin/build.sh + touch packages/dummy - name: Package upload uses: forgejo/upload-artifact@v3 with: diff --git a/.forgejo/workflows/check-backports.yml b/.forgejo/workflows/check-backports.yml new file mode 100644 index 0000000..b9f76a6 --- /dev/null +++ b/.forgejo/workflows/check-backports.yml @@ -0,0 +1,28 @@ +on: + workflow_dispatch: + + schedule: + - cron: '0 5 * * *' + +jobs: + check-backports: + name: Check backports repo + runs-on: x86_64 + container: + image: alpine:latest + env: + downstream: https://ayakael.net/api/packages/forge/alpine/v3.21/backports + FORGEJO_TOKEN: ${{ secrets.forgejo_token }} + LABEL_NUMBER: 1 + ALL_PACKAGES: true + steps: + - name: Environment setup + run: apk add grep coreutils gawk curl wget bash nodejs git jq sed + - name: Get scripts + uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: Check out-of-date packages + run: ${{ github.workspace }}/.forgejo/bin/check_ver.sh + - name: Create issues + run: ${{ github.workspace }}/.forgejo/bin/create_issue.sh diff --git a/.forgejo/workflows/check-community.yml b/.forgejo/workflows/check-community.yml new file mode 100644 index 0000000..9385687 --- /dev/null +++ b/.forgejo/workflows/check-community.yml @@ -0,0 +1,27 @@ +on: + workflow_dispatch: + + schedule: + - cron: '0 5 * * *' + +jobs: + check-community: + name: Check community repo + runs-on: x86_64 + container: + image: alpine:latest + env: + downstream: https://dl-cdn.alpinelinux.org/alpine/edge/community + FORGEJO_TOKEN: ${{ secrets.forgejo_token }} + LABEL_NUMBER: 4 + steps: + - name: Environment setup + run: apk add grep coreutils gawk curl wget bash nodejs git jq sed + - name: Get scripts + uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: Check out-of-date packages + run: ${{ github.workspace }}/.forgejo/bin/check_ver.sh + - name: Create issues + run: ${{ github.workspace }}/.forgejo/bin/create_issue.sh diff --git a/.forgejo/workflows/check-testing.yml b/.forgejo/workflows/check-testing.yml new file mode 100644 index 0000000..2b8f7ed --- /dev/null +++ b/.forgejo/workflows/check-testing.yml @@ -0,0 +1,27 @@ +on: + workflow_dispatch: + + schedule: + - cron: '0 5 * * *' + +jobs: + check-community: + name: Check testing repo + runs-on: x86_64 + container: + image: alpine:latest + env: + downstream: https://dl-cdn.alpinelinux.org/alpine/edge/testing + FORGEJO_TOKEN: ${{ secrets.forgejo_token }} + LABEL_NUMBER: 4 + steps: + - name: Environment setup + run: apk add grep coreutils gawk curl wget bash nodejs git jq sed + - name: Get scripts + uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: Check out-of-date packages + run: ${{ github.workspace }}/.forgejo/bin/check_ver.sh + - name: Create issues + run: ${{ github.workspace }}/.forgejo/bin/create_issue.sh diff --git a/.forgejo/workflows/check-user.yml b/.forgejo/workflows/check-user.yml new file mode 100644 index 0000000..09b0f3c --- /dev/null +++ b/.forgejo/workflows/check-user.yml @@ -0,0 +1,27 @@ +on: + workflow_dispatch: + + schedule: + - cron: '0 5 * * *' + +jobs: + check-user: + name: Check user repo + runs-on: x86_64 + container: + image: alpine:latest + env: + downstream: https://ayakael.net/api/packages/forge/alpine/edge/user + FORGEJO_TOKEN: ${{ secrets.forgejo_token }} + LABEL_NUMBER: 4 + steps: + - name: Environment setup + run: apk add grep coreutils gawk curl wget bash nodejs git jq sed + - name: Get scripts + uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: Check out-of-date packages + run: ${{ github.workspace }}/.forgejo/bin/check_ver.sh + - name: Create issues + run: ${{ github.workspace }}/.forgejo/bin/create_issue.sh