pmaports/.ci/testcases/test_source.py

#!/usr/bin/env python3
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
# Various checks on source= in the APKBUILDs

import glob
import logging
import os
import pytest
import sys

import add_pmbootstrap_to_import_path
import pmb.parse
import pmb.parse._apkbuild
import pmb.parse.apkindex
import pmb.helpers.repo


def parse_source_from_checksums(args, apkbuild_path):
    """
    Read the APKBUILD file and parse source files from the checksums at the
    bottom. This has always the same format, even if $source is built with hard
    to parse shell code (like in postmarketos-base).

    :param apkbuild_path: full path to the APKBUILD
    :returns: dict of parsed "source" filenames and checksums, e.g.:
              {"first.patch": "b4dc4f3…",
               "second.patch": "b4dc4f3…"}
    """
    start = 'sha512sums="'
    in_block = False
    ret = {}

    with open(apkbuild_path, encoding="utf-8") as handle:
        for line in handle.readlines():
            # Find start
            if not in_block:
                if line.startswith(start):
                    in_block = True
                else:
                    continue

            # sha512sums may have lines without checksums:
            # https://gitlab.alpinelinux.org/alpine/abuild/-/merge_requests/73
            if "  " not in line:
                continue

            try:
                checksum, filename = line.rstrip().split("  ", 2)
            except ValueError:
                raise ValueError("Failed to parse checksums. Try to delete the"
                                 " checksums and generate them again with"
                                 f" 'pmbootstrap checksum': {apkbuild_path}")

            # Cut off 'sha512sums="' if the first checksum is in that line
            if checksum.startswith(start):
                checksum = checksum[len(start):]

            # Find end
            if filename.endswith('"'):
                filename = filename[:-1]

            ret[filename] = checksum
    return ret


def test_aports_unreferenced_files(args):
    """
    Raise an error if an unreferenced file is found
    """
    for apkbuild_path in glob.iglob(args.aports + "/**/APKBUILD", recursive=True):
        # pmbootstrap parser has some issues with complicated APKBUILDs, skip those.
        if apkbuild_path.startswith(args.aports + "/cross/"):
            continue

        apkbuild = pmb.parse.apkbuild(apkbuild_path)
        sources_chk = parse_source_from_checksums(args, apkbuild_path)

        # Collect files from subpackages
        subpackage_installs = []
        subpackage_triggers = []
        if apkbuild["subpackages"]:
            for subpackage in apkbuild["subpackages"].values():
                if not subpackage:
                    continue
                subpackage_installs += subpackage.get("install", [])
                subpackage_triggers += subpackage.get("triggers", [])

        # Collect trigger files
        trigger_sources = []
        for trigger in apkbuild["triggers"] + subpackage_triggers:
            trigger_sources.append(trigger.split("=")[0])

        dirname = os.path.dirname(apkbuild_path)
        for file in glob.iglob(dirname + "/**", recursive=True):
            rel_file_path = os.path.relpath(file, dirname)
            # Skip APKBUILDs and directories
            if rel_file_path == "APKBUILD" or os.path.isdir(file):
                continue

            if os.path.basename(rel_file_path) not in sources_chk \
                    and rel_file_path not in apkbuild["install"] \
                    and rel_file_path not in subpackage_installs \
                    and rel_file_path not in trigger_sources:
                raise RuntimeError(f"{apkbuild_path}: found unreferenced file: {rel_file_path}")


def test_distfiles_conflict(args):
    """
    Make sure that each filename mentioned in any source= of any APKBUILD
    always has the same checksum. This is important because apk caches
    downloaded source files in a flat distfiles directory. So if two APKBUILDs
    both download a file with the same filename but different checksum, and the
    user builds both after each other, abuild will fail on the second build
    with a checksum error.
    """
    source_all = {}
    for apkbuild_path in glob.iglob(f"{args.aports}/**/APKBUILD", recursive=True):
        source = parse_source_from_checksums(args, apkbuild_path)
        dir_path = os.path.dirname(apkbuild_path)
        apkbuild_rel = os.path.relpath(apkbuild_path, args.aports)
        for filename, checksum in source.items():
            # Files bundled with the APKBUILD don't get copied to the distfiles
            # cache, so not relevant for this check. Use glob.glob here and not
            # iglob, because we don't want an iterator.
            if glob.glob(f"{dir_path}/**/{filename}", recursive=True):
                continue

            # First time seeing this file
            if filename not in source_all:
                source_all[filename] = {"checksum": checksum,
                                        "apkbuild_rel": apkbuild_rel}
                continue

            # Saw this file already with same checksum
            if checksum == source_all[filename]["checksum"]:
                continue

            # Saw this file already with different checksum
            logging.error("")
            logging.error(f"ERROR: the source file '{filename}' has different"
                          " checksums in the following files:")
            logging.error(f"- {source_all[filename]['apkbuild_rel']}:")
            logging.error(f"  {source_all[filename]['checksum']}")
            logging.error(f"- {apkbuild_rel}:")
            logging.error(f"  {checksum}")
            logging.error("")
            logging.error("Fix this by setting a different target filename in"
                          " the package you modified:")
            logging.error("https://wiki.alpinelinux.org/wiki/APKBUILD_Reference#source")
            logging.error("")
            raise RuntimeError(f"Conflict with source file '{filename}'")