CI: implement distfile-check as python test (MR 3608)

Reimplement the distfile-check as python test inside this git
repository, so we don't need to download a tarball of ci-common.git to
run it. This would not have been nice for running the test with
'pmbootstrap ci', as we don't want it to get downloaded every time the
test runs.

This new implementation is done in less than 40 lines of code, very fast
and doesn't need to create a lot of files in /tmp to build a
"distfiletree".

.ci/testcases/test_source.py

@@ -4,9 +4,10 @@
 # Various checks on source= in the APKBUILDs
 
 import glob
+import logging
+import os
 import pytest
 import sys
-import os
 
 import add_pmbootstrap_to_import_path
 import pmb.parse
@@ -102,3 +103,50 @@ def test_aports_unreferenced_files(args):
                     and rel_file_path not in subpackage_installs \
                     and rel_file_path not in trigger_sources:
                 raise RuntimeError(f"{apkbuild_path}: found unreferenced file: {rel_file_path}")
+
+
+def test_distfiles_conflict(args):
+    """
+    Make sure that each filename mentioned in any source= of any APKBUILD
+    always has the same checksum. This is important because apk caches
+    downloaded source files in a flat distfiles directory. So if two APKBUILDs
+    both download a file with the same filename but different checksum, and the
+    user builds both after each other, abuild will fail on the second build
+    with a checksum error.
+    """
+    source_all = {}
+    for apkbuild_path in glob.iglob(f"{args.aports}/**/APKBUILD", recursive=True):
+        source = parse_source_from_checksums(args, apkbuild_path)
+        dir_path = os.path.dirname(apkbuild_path)
+        apkbuild_rel = os.path.relpath(apkbuild_path, args.aports)
+        for filename, checksum in source.items():
+            # Files bundled with the APKBUILD don't get copied to the distfiles
+            # cache, so not relevant for this check. Use glob.glob here and not
+            # iglob, because we don't want an iterator.
+            if glob.glob(f"{dir_path}/**/{filename}", recursive=True):
+                continue
+
+            # First time seeing this file
+            if filename not in source_all:
+                source_all[filename] = {"checksum": checksum,
+                                        "apkbuild_rel": apkbuild_rel}
+                continue
+
+            # Saw this file already with same checksum
+            if checksum == source_all[filename]["checksum"]:
+                continue
+
+            # Saw this file already with different checksum
+            logging.error("")
+            logging.error(f"ERROR: the source file '{filename}' has different"
+                          " checksums in the following files:")
+            logging.error(f"- {source_all[filename]['apkbuild_rel']}:")
+            logging.error(f"  {source_all[filename]['checksum']}")
+            logging.error(f"- {apkbuild_rel}:")
+            logging.error(f"  {checksum}")
+            logging.error("")
+            logging.error("Fix this by setting a different target filename in"
+                          " the package you modified:")
+            logging.error("https://wiki.alpinelinux.org/wiki/APKBUILD_Reference#source")
+            logging.error("")
+            raise RuntimeError(f"Conflict with source file '{filename}'")

.gitlab-ci.yml

@@ -85,18 +85,6 @@ pytest-commits:
       - pmbootstrap.cfg
     expire_in: 1 week
 
-distfile-check:
-  stage: lint
-  <<: *only-default
-  before_script:
-    - apk -q add make findutils
-    - wget "https://gitlab.com/postmarketOS/ci-common/-/archive/master/ci-common-master.tar.bz2"
-    - tar -xf ci-common-master.tar.bz2
-    - mv ci-common-master/distfile-check/*.sh ci-common-master/distfile-check/Makefile .
-    - rm -r ci-common-master ci-common-master.tar.bz2
-  script:
-    - make -j999
-
 # APKBUILD linting
 aport-lint:
   stage: lint