a1242ed35e
Reimplement the distfile-check as python test inside this git repository, so we don't need to download a tarball of ci-common.git to run it. This would not have been nice for running the test with 'pmbootstrap ci', as we don't want it to get downloaded every time the test runs. This new implementation is done in less than 40 lines of code, very fast and doesn't need to create a lot of files in /tmp to build a "distfiletree".
152 lines
6 KiB
Python
152 lines
6 KiB
Python
#!/usr/bin/env python3
|
|
# Copyright 2022 Oliver Smith
|
|
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
# Various checks on source= in the APKBUILDs
|
|
|
|
import glob
|
|
import logging
|
|
import os
|
|
import pytest
|
|
import sys
|
|
|
|
import add_pmbootstrap_to_import_path
|
|
import pmb.parse
|
|
import pmb.parse._apkbuild
|
|
import pmb.parse.apkindex
|
|
import pmb.helpers.repo
|
|
|
|
|
|
def parse_source_from_checksums(args, apkbuild_path):
|
|
"""
|
|
Read the APKBUILD file and parse source files from the checksums at the
|
|
bottom. This has always the same format, even if $source is built with hard
|
|
to parse shell code (like in postmarketos-base).
|
|
|
|
:param apkbuild_path: full path to the APKBUILD
|
|
:returns: dict of parsed "source" filenames and checksums, e.g.:
|
|
{"first.patch": "b4dc4f3…",
|
|
"second.patch": "b4dc4f3…"}
|
|
"""
|
|
start = 'sha512sums="'
|
|
in_block = False
|
|
ret = {}
|
|
|
|
with open(apkbuild_path, encoding="utf-8") as handle:
|
|
for line in handle.readlines():
|
|
# Find start
|
|
if not in_block:
|
|
if line.startswith(start):
|
|
in_block = True
|
|
else:
|
|
continue
|
|
|
|
# sha512sums may have lines without checksums:
|
|
# https://gitlab.alpinelinux.org/alpine/abuild/-/merge_requests/73
|
|
if " " not in line:
|
|
continue
|
|
|
|
try:
|
|
checksum, filename = line.rstrip().split(" ", 2)
|
|
except ValueError:
|
|
raise ValueError("Failed to parse checksums. Try to delete the"
|
|
" checksums and generate them again with"
|
|
f" 'pmbootstrap checksum': {apkbuild_path}")
|
|
|
|
# Cut off 'sha512sums="' if the first checksum is in that line
|
|
if checksum.startswith(start):
|
|
checksum = checksum[len(start):]
|
|
|
|
# Find end
|
|
if filename.endswith('"'):
|
|
filename = filename[:-1]
|
|
|
|
ret[filename] = checksum
|
|
return ret
|
|
|
|
|
|
def test_aports_unreferenced_files(args):
|
|
"""
|
|
Raise an error if an unreferenced file is found
|
|
"""
|
|
for apkbuild_path in glob.iglob(args.aports + "/**/APKBUILD", recursive=True):
|
|
# pmbootstrap parser has some issues with complicated APKBUILDs, skip those.
|
|
if apkbuild_path.startswith(args.aports + "/cross/"):
|
|
continue
|
|
|
|
apkbuild = pmb.parse.apkbuild(apkbuild_path)
|
|
sources_chk = parse_source_from_checksums(args, apkbuild_path)
|
|
|
|
# Collect files from subpackages
|
|
subpackage_installs = []
|
|
subpackage_triggers = []
|
|
if apkbuild["subpackages"]:
|
|
for subpackage in apkbuild["subpackages"].values():
|
|
if not subpackage:
|
|
continue
|
|
subpackage_installs += subpackage.get("install", [])
|
|
subpackage_triggers += subpackage.get("triggers", [])
|
|
|
|
# Collect trigger files
|
|
trigger_sources = []
|
|
for trigger in apkbuild["triggers"] + subpackage_triggers:
|
|
trigger_sources.append(trigger.split("=")[0])
|
|
|
|
dirname = os.path.dirname(apkbuild_path)
|
|
for file in glob.iglob(dirname + "/**", recursive=True):
|
|
rel_file_path = os.path.relpath(file, dirname)
|
|
# Skip APKBUILDs and directories
|
|
if rel_file_path == "APKBUILD" or os.path.isdir(file):
|
|
continue
|
|
|
|
if os.path.basename(rel_file_path) not in sources_chk \
|
|
and rel_file_path not in apkbuild["install"] \
|
|
and rel_file_path not in subpackage_installs \
|
|
and rel_file_path not in trigger_sources:
|
|
raise RuntimeError(f"{apkbuild_path}: found unreferenced file: {rel_file_path}")
|
|
|
|
|
|
def test_distfiles_conflict(args):
|
|
"""
|
|
Make sure that each filename mentioned in any source= of any APKBUILD
|
|
always has the same checksum. This is important because apk caches
|
|
downloaded source files in a flat distfiles directory. So if two APKBUILDs
|
|
both download a file with the same filename but different checksum, and the
|
|
user builds both after each other, abuild will fail on the second build
|
|
with a checksum error.
|
|
"""
|
|
source_all = {}
|
|
for apkbuild_path in glob.iglob(f"{args.aports}/**/APKBUILD", recursive=True):
|
|
source = parse_source_from_checksums(args, apkbuild_path)
|
|
dir_path = os.path.dirname(apkbuild_path)
|
|
apkbuild_rel = os.path.relpath(apkbuild_path, args.aports)
|
|
for filename, checksum in source.items():
|
|
# Files bundled with the APKBUILD don't get copied to the distfiles
|
|
# cache, so not relevant for this check. Use glob.glob here and not
|
|
# iglob, because we don't want an iterator.
|
|
if glob.glob(f"{dir_path}/**/{filename}", recursive=True):
|
|
continue
|
|
|
|
# First time seeing this file
|
|
if filename not in source_all:
|
|
source_all[filename] = {"checksum": checksum,
|
|
"apkbuild_rel": apkbuild_rel}
|
|
continue
|
|
|
|
# Saw this file already with same checksum
|
|
if checksum == source_all[filename]["checksum"]:
|
|
continue
|
|
|
|
# Saw this file already with different checksum
|
|
logging.error("")
|
|
logging.error(f"ERROR: the source file '{filename}' has different"
|
|
" checksums in the following files:")
|
|
logging.error(f"- {source_all[filename]['apkbuild_rel']}:")
|
|
logging.error(f" {source_all[filename]['checksum']}")
|
|
logging.error(f"- {apkbuild_rel}:")
|
|
logging.error(f" {checksum}")
|
|
logging.error("")
|
|
logging.error("Fix this by setting a different target filename in"
|
|
" the package you modified:")
|
|
logging.error("https://wiki.alpinelinux.org/wiki/APKBUILD_Reference#source")
|
|
logging.error("")
|
|
raise RuntimeError(f"Conflict with source file '{filename}'")
|