From 57744b984ee4c03e2f8317472b18719aa51dcc4f Mon Sep 17 00:00:00 2001 From: Caleb Connolly Date: Mon, 18 Dec 2023 22:48:14 +0000 Subject: [PATCH] postmarketos-initramfs: export logs on boot failure (MR 4646) Currently, when postmarketOS fails to boot up, retrieving any information necessary to root cause the issue is non-trivial, requiring building a custom initramfs with debug-shell enabled and then manually copying out data. Let's improve the situation by exporting logs automatically on boot failure. This is safer than just exposing a root shell but still provides a whole lot of useful info that should make duplicating and triaging issues much easier for developers. This commit implements the functionality and includes some generally useful data. We then generate a README and expose files for each command/log as well as an archive that can be easily attached to a GitLab issue. To help with triaging, also record the version of the postmarketos-initramfs package and hash the init.sh and init_functions.sh files. For testing purposes, you can trigger the log recovery mode on-time by creating an empty file named ".pmos_export_logs" in the /boot partition. Signed-off-by: Caleb Connolly --- main/postmarketos-initramfs/APKBUILD | 16 ++- main/postmarketos-initramfs/init.sh | 20 +++- main/postmarketos-initramfs/init_functions.sh | 111 ++++++++++++++++-- .../20-debug-shell.sh | 10 +- .../APKBUILD | 4 +- .../00-maximum-attention.sh | 2 +- .../APKBUILD | 8 +- .../APKBUILD | 4 +- .../netboot.sh | 2 +- 9 files changed, 150 insertions(+), 27 deletions(-) diff --git a/main/postmarketos-initramfs/APKBUILD b/main/postmarketos-initramfs/APKBUILD index 3e424cc85..7f5bddcd9 100644 --- a/main/postmarketos-initramfs/APKBUILD +++ b/main/postmarketos-initramfs/APKBUILD @@ -1,7 +1,7 @@ # Maintainer: Oliver Smith # Co-Maintainer: Clayton Craft pkgname=postmarketos-initramfs -pkgver=2.3.0 +pkgver=2.4.0 pkgrel=0 pkgdesc="Base files for the postmarketOS initramfs / initramfs-extra" url="https://postmarketos.org" @@ -39,6 +39,14 @@ source=" arch="noarch" license="GPL-2.0-or-later" +build() { + mkdir -p "$builddir" + + # Replace <> with the actual version + sed "s|<>|$pkgver-r$pkgrel|" \ + "$srcdir/init.sh" > "$builddir/init.sh" +} + package() { install -Dm644 "$srcdir/unudhcpd.conf" \ "$pkgdir/etc/unudhcpd.conf" @@ -46,7 +54,7 @@ package() { install -Dm644 "$srcdir/init_functions.sh" \ "$pkgdir/usr/share/initramfs/init_functions.sh" - install -Dm755 "$srcdir/init.sh" \ + install -Dm755 "$builddir/init.sh" \ "$pkgdir/usr/share/initramfs/init.sh" install -Dm644 "$srcdir"/00-initramfs-base.dirs \ @@ -79,8 +87,8 @@ sha512sums=" 5b364300f31c91fd0591eb0715f67cbf5383f45246a5fb9f34b79f7cb2e3b15768b2130e5f32f816cc169950f988c1beabc879ba31645c58ce131a288dbc071d 00-initramfs-base.dirs ab41b45b0613f25a61114ed8c8b92bc53c60838f6e2e0ba18c76e5369b2984e6023a0661887692673aca3f647f268c468a468f6b1ac424cfee609017a89481dd 00-initramfs-base.files 8a4adad3785af474b36a09a05f6a3b2c4b4f43aac331a53b903abfa51ea12be1e3d1d807b7a6e66a1346815f3b0044daf8cd62e21e2dc75d2db13ee265a72985 00-initramfs-extra-base.files -701b6252a536dfc7bddef0f5c3ccb16ad680f70efbad872b74d61ef24c98edcf623b19b5e1a2973bc3fa66ca009a515d7ba3563f0d417306a0a2b5680bee7efd init.sh -74cfd8e30811bcf3bb075bcc4db0b59c4f63ba83552cfe371132dacd4c41d73c04c65d309609578369243e9b969feba56b4bdd478eb5e7ebff7099e84f7c0f01 init_functions.sh +4bfcb41b2d5ccd577a6203a9479e5bdf056064d20f02f9120f1a074c7c280c0231cf380ed6be7c80733ed4ae101385c14f123a8a41780bbb7d0851387cd9c613 init.sh +58ee2b36b858c9c70fabbcb9a12a754afc97e6dfe09ddb7a2ec02c6fb593cc40677ae7049ba5ffda7797cba20e4d35f0d320c5fb4d3072cdbdf172cce6665bb0 init_functions.sh ba3275a9af788c7c782322a22a0f144d5e50e3498ea6886486a29331f23ae89cd32d500a3635cfa7cab369afba92edc18aeca64ccbf0cd589061cce23d15b46c unudhcpd.conf 675e7d5bee39b2df7d322117f8dcaccc274d61beaf4d50ead19bbf2109446d64b1c0aa0c5b4f9846eb6c1c403418f28f6364eff4537ba41120fbfcbc484b7da7 mdev.conf " diff --git a/main/postmarketos-initramfs/init.sh b/main/postmarketos-initramfs/init.sh index 086d2cb53..b9518f4a0 100644 --- a/main/postmarketos-initramfs/init.sh +++ b/main/postmarketos-initramfs/init.sh @@ -14,6 +14,9 @@ IN_CI="false" # provide a default for os-release's VERSION in case the file doesn't exist VERSION="${VERSION:-unknown}" +# This is set during packaging and is used when triaging bug reports +INITRAMFS_PKG_VERSION="<>" + export PATH=/usr/bin:/bin:/usr/sbin:/sbin /bin/busybox --install -s /bin/busybox-extras --install -s @@ -40,7 +43,7 @@ run_hooks /hooks if [ "$IN_CI" = "true" ]; then echo "PMOS: CI tests done, disabling console and looping forever" dmesg -n 1 - loop_forever + fail_halt_boot fi # Always run dhcp daemon/usb networking for now (later this should only @@ -54,6 +57,19 @@ extract_initramfs_extra /boot/initramfs-extra setup_udev run_hooks /hooks-extra +# For testing the mass storage gadget log export function. We use a flag +# file on /boot so that we can test it on all devices as modifying the +# kernel cmdline is not always possible. +if [ -e /boot/.pmos_export_logs ]; then + echo "PMOS: Exporting logs via mass storage gadget" + show_splash "Exporting boot logs..." + # Delete the flag so we don't soft-brick the device by always booting + # to the log export mode. + mount -o remount,rw /boot + rm -f /boot/.pmos_export_logs + fail_halt_boot +fi + wait_root_partition delete_old_install_partition resize_root_partition @@ -77,4 +93,4 @@ exec switch_root /sysroot "$init" echo "ERROR: switch_root failed!" echo "Looping forever. Install and use the debug-shell hook to debug this." echo "For more information, see " -loop_forever +fail_halt_boot diff --git a/main/postmarketos-initramfs/init_functions.sh b/main/postmarketos-initramfs/init_functions.sh index 5b60ee45f..6e6cbc9f0 100644 --- a/main/postmarketos-initramfs/init_functions.sh +++ b/main/postmarketos-initramfs/init_functions.sh @@ -304,7 +304,7 @@ mount_boot_partition() { if [ -z "$partition" ]; then echo "ERROR: boot partition not found!" show_splash "ERROR: Boot partition not found\\nhttps://postmarketos.org/troubleshooting" - loop_forever + fail_halt_boot fi if [ "$2" = "rw" ]; then @@ -341,7 +341,7 @@ extract_initramfs_extra() { if [ ! -e "$initramfs_extra" ]; then echo "ERROR: initramfs-extra not found!" show_splash "ERROR: initramfs-extra not found\\nhttps://postmarketos.org/troubleshooting" - loop_forever + fail_halt_boot fi echo "Extract $initramfs_extra" # uncompressed: @@ -513,7 +513,7 @@ mount_root_partition() { if ! { [ "$type" = "ext4" ] || [ "$type" = "f2fs" ] || [ "$type" = "btrfs" ]; } then echo "ERROR: Detected unsupported '$type' filesystem ($partition)." show_splash "ERROR: unsupported '$type' filesystem ($partition)\\nhttps://postmarketos.org/troubleshooting" - loop_forever + fail_halt_boot fi if ! modprobe "$type"; then @@ -522,13 +522,13 @@ mount_root_partition() { if ! mount -t "$type" -o ro"$rootfsopts" "$partition" /sysroot; then echo "ERROR: unable to mount root partition!" show_splash "ERROR: unable to mount root partition\\nhttps://postmarketos.org/troubleshooting" - loop_forever + fail_halt_boot fi if ! [ -e /sysroot/usr ]; then echo "ERROR: root partition appeared to mount but does not contain a root filesystem!" show_splash "ERROR: root partition does not contain a root filesystem\\nhttps://postmarketos.org/troubleshooting" - loop_forever + fail_halt_boot fi } @@ -584,9 +584,11 @@ setup_usb_configfs_udc() { echo "$_udc_dev" > /config/usb_gadget/g1/UDC || echo " Couldn't write new UDC" } +# $1: if set, skip writing to the UDC setup_usb_network_configfs() { # See: https://www.kernel.org/doc/Documentation/usb/gadget_configfs.txt CONFIGFS=/config/usb_gadget + local skip_udc="$1" if ! [ -e "$CONFIGFS" ]; then echo " /config/usb_gadget does not exist, skipping configfs usb gadget" @@ -638,7 +640,11 @@ setup_usb_network_configfs() { ln -s $CONFIGFS/g1/functions/"$usb_network_function" $CONFIGFS/g1/configs/c.1 \ || echo " Couldn't symlink $usb_network_function" - setup_usb_configfs_udc + # If an argument was supplied then skip writing to the UDC (only used for mass storage + # log recovery) + if [ -z "$skip_udc" ]; then + setup_usb_configfs_udc + fi } setup_usb_network() { @@ -771,7 +777,98 @@ setup_bootchart2() { fi } -loop_forever() { +mkhash() { + sha256sum "$1" | cut -d " " -f 1 +} + +# Create a small disk image and copy logs to it so they can be exposed via mass storage +create_logs_disk() { + local loop_dev="$1" + local upload_file="" + echo "Creating logs disk" + + fallocate -l 10M /tmp/logs.img + # The log device used is assumed to be $loop_dev + losetup -f /tmp/logs.img + mkfs.vfat -n "PMOS_LOGS" "$loop_dev" + mkdir -p /tmp/logs + mount "$loop_dev" /tmp/logs + + # Copy logs + cp /pmOS_init.log /tmp/logs/pmOS_init.txt + dmesg > /tmp/logs/dmesg.txt + blkid > /tmp/logs/blkid.txt + cat /proc/cmdline > /tmp/logs/cmdline.txt + cat /proc/partitions > /tmp/logs/partitions.txt + # Include FDT if it exists + [ -e /sys/firmware/fdt ] && cp /sys/firmware/fdt /tmp/logs/fdt.dtb + + # Additional info about the initramfs + { + echo "initramfs-version: $INITRAMFS_PKG_VERSION" + # Take hashes of the initramfs files so we can be sure they weren't modified inadvertantly + echo "init-hash: $(mkhash /init)" + echo "init-functions-hash: $(mkhash /init_functions.sh)" + } >> /tmp/logs/_info + + # Create a tar file with all the logs. We don't include the date because on many devices + # (especially Qualcomm) the RTC is likely wrong. + upload_file="${deviceinfo_codename}-${VERSION}-$(uname -r).tar.gz" + # Done in a subshell to not change the working directory of init + (cd /tmp/logs || ( echo "Couldn't cd to /tmp/logs"; return ); tar -cv ./* | gzip -6 -c > "/tmp/$upload_file") + mv "/tmp/$upload_file" /tmp/logs/ + + # Create a README with instructions on how to report an issue + cat > /tmp/logs/README.txt <<-EOF + Something went wrong and your device did not boot properly. If this was unexpected + then please open a new issue by visiting + + https://gitlab.com/postmarketOS/pmaports/-/issues/new + + and attach the following file by dragging it onto the page: + + * $upload_file + + You are running postmarketOS $VERSION on kernel $(uname -r). + EOF + + # Unmount + umount /tmp/logs +} + +# Make logs available via mass storage gadget +export_logs() { + local loop_dev="" + usb_mass_storage_function="mass_storage.0" + active_udc="$(cat /config/usb_gadget/g1/UDC)" + + loop_dev="$(losetup -f)" + + create_logs_disk "$loop_dev" + + echo "Making logs available via mass storage" + + # Set up network gadget if not already done + if [ -z "$active_udc" ]; then + setup_usb_network_configfs "skip_udc" + else + # Unset UDC + echo "" > /config/usb_gadget/g1/UDC + fi + + mkdir "$CONFIGFS"/g1/functions/"$usb_mass_storage_function" || return + + echo "$loop_dev" > "$CONFIGFS"/g1/functions/"$usb_mass_storage_function"/lun.0/file + + ln -s "$CONFIGFS"/g1/functions/"$usb_mass_storage_function" \ + "$CONFIGFS"/g1/configs/c.1 || return + + setup_usb_configfs_udc +} + +fail_halt_boot() { + export_logs + echo "Looping forever" while true; do sleep 1 done diff --git a/main/postmarketos-mkinitfs-hook-debug-shell/20-debug-shell.sh b/main/postmarketos-mkinitfs-hook-debug-shell/20-debug-shell.sh index d5cc3a743..59496c503 100644 --- a/main/postmarketos-mkinitfs-hook-debug-shell/20-debug-shell.sh +++ b/main/postmarketos-mkinitfs-hook-debug-shell/20-debug-shell.sh @@ -15,7 +15,7 @@ echo "Create 'pmos_continue_boot' script" #Disable any active usb mass storage echo "if [ -d /config/usb_gadget/g1/functions/mass_storage.0 ]; then setup_usb_storage; fi" echo "pkill -9 -f pmos_shell" - echo "pkill -f pmos_loop_forever" + echo "pkill -f pmos_fail_halt_boot" echo "pkill -f telnetd.*:${TELNET_PORT}" } >/usr/bin/pmos_continue_boot chmod +x /usr/bin/pmos_continue_boot @@ -27,13 +27,13 @@ echo "Create 'pmos_shell' script" } >/usr/bin/pmos_shell chmod +x /usr/bin/pmos_shell -echo "Create 'pmos_loop_forever' script" +echo "Create 'pmos_fail_halt_boot' script" { echo "#!/bin/sh" echo '. /init_functions.sh' - echo "loop_forever" -} >/usr/bin/pmos_loop_forever -chmod +x /usr/bin/pmos_loop_forever + echo "fail_halt_boot" +} >/usr/bin/pmos_fail_halt_boot +chmod +x /usr/bin/pmos_fail_halt_boot echo "Start the telnet daemon" { diff --git a/main/postmarketos-mkinitfs-hook-debug-shell/APKBUILD b/main/postmarketos-mkinitfs-hook-debug-shell/APKBUILD index 71414bf2f..46d017c22 100644 --- a/main/postmarketos-mkinitfs-hook-debug-shell/APKBUILD +++ b/main/postmarketos-mkinitfs-hook-debug-shell/APKBUILD @@ -1,5 +1,5 @@ pkgname=postmarketos-mkinitfs-hook-debug-shell -pkgver=0.5.0 +pkgver=0.6.0 pkgrel=0 pkgdesc="Root shell in the initramfs (security hole, for debugging only)" url="https://postmarketos.org" @@ -19,7 +19,7 @@ package() { } sha512sums=" -0750a33f3b68fd6f751bdcf3c59656ec8c8af60e13c25b13394937134d4cbeaacd237b5d92e2a978319f705781f39712c71673367cec52b584f666af50b6f952 20-debug-shell.sh +add95b3fa64b804fd386298406fbc55489e3e122d70f40e45627a450d299e148a346a4a147c2b28e0110ecef14e9e0b62ac75a0bd2d39295c2407ebb5e93708c 20-debug-shell.sh a739bc47d905d189edb26d9ebfd062023720fefdaab27207471c16d53a9c12ea8b81092a1047d8f2300e42ba500bdf6c5a3343aca55aab5bf8e84d68eb5680ab 20-debug-shell.files 75d485c2e9f352cfd717b7a92753a9dfc4a72526a44bcbb784eacb4ef9011072b3ffa1c42a317c0940598cc076fb6c61676c440e5b188378b19ca08d882c1338 setup_usb_storage.sh " diff --git a/main/postmarketos-mkinitfs-hook-maximum-attention/00-maximum-attention.sh b/main/postmarketos-mkinitfs-hook-maximum-attention/00-maximum-attention.sh index 2d628cc50..671c7bc4b 100644 --- a/main/postmarketos-mkinitfs-hook-maximum-attention/00-maximum-attention.sh +++ b/main/postmarketos-mkinitfs-hook-maximum-attention/00-maximum-attention.sh @@ -53,4 +53,4 @@ vibrate_loop() { blink_leds $(find_leds) & vibrate_loop $(find_vibrator) & -loop_forever +fail_halt_boot diff --git a/main/postmarketos-mkinitfs-hook-maximum-attention/APKBUILD b/main/postmarketos-mkinitfs-hook-maximum-attention/APKBUILD index 6f0d4df69..fbb5179ac 100644 --- a/main/postmarketos-mkinitfs-hook-maximum-attention/APKBUILD +++ b/main/postmarketos-mkinitfs-hook-maximum-attention/APKBUILD @@ -1,6 +1,6 @@ pkgname=postmarketos-mkinitfs-hook-maximum-attention -pkgver=0.1.0 -pkgrel=1 +pkgver=0.2.0 +pkgrel=0 pkgdesc="Script to activate all user-visible outputs from the initramfs (to confirm working kernel, for debugging only)" url="https://postmarketos.org" depends="postmarketos-mkinitfs" @@ -12,4 +12,6 @@ package() { install -Dm644 "$srcdir"/00-maximum-attention.sh \ -t "$pkgdir"/usr/share/mkinitfs/hooks/ } -sha512sums="c29cc41649c284c0ec8201715cd9452f838801a22ea6c2a312ea87a58c61c1caa05a3b1ff33ce4a4d680a95e79fb10108308b72173f1fa97bac5cefde7ab30ee 00-maximum-attention.sh" +sha512sums=" +ca3ef48e76e21fdb67fdf7477fb4739e329ce6ea04a52ac14e9365adb92058b21e08fcbc4020519c61ba20cd3c91d254118ece3d1818cb46f86e86f5ffa1b90a 00-maximum-attention.sh +" diff --git a/main/postmarketos-mkinitfs-hook-netboot/APKBUILD b/main/postmarketos-mkinitfs-hook-netboot/APKBUILD index e4cd01bf4..c73cc2856 100644 --- a/main/postmarketos-mkinitfs-hook-netboot/APKBUILD +++ b/main/postmarketos-mkinitfs-hook-netboot/APKBUILD @@ -1,6 +1,6 @@ # Maintainer: Mark pkgname=postmarketos-mkinitfs-hook-netboot -pkgver=4.0.3 +pkgver=4.1.3 pkgrel=0 pkgdesc="Netboot postmarketOS using nbd feature" url="https://postmarketos.org" @@ -21,6 +21,6 @@ package() { } sha512sums=" -039b6ae08ffb2e24573a5659fef8ff14f70219e06fe6dd6c5ad459f3b54883e17166060813762cdc71b7c774f450956158871fd64c70330d0a661a8cc94666e7 netboot.sh +c681d0c39eecd9fe5d284e07f9c6f79e41085628abe1c645b8ea3cc4e621fd603fca9db4a555fa689a3268925cb6d23c12baa98986baed452375acd1e3fe26b1 netboot.sh 3c9c812a6cd25640e6cf4d722cff6c6356d5666ec6009b4c65f89e62211cc406d2cd09e0fdda6b1c1906ee32c3854988ab701cee8811208e3ef666f09e638568 netboot.modules " diff --git a/main/postmarketos-mkinitfs-hook-netboot/netboot.sh b/main/postmarketos-mkinitfs-hook-netboot/netboot.sh index d668302be..8ac2dfc34 100644 --- a/main/postmarketos-mkinitfs-hook-netboot/netboot.sh +++ b/main/postmarketos-mkinitfs-hook-netboot/netboot.sh @@ -17,7 +17,7 @@ modprobe nbd if [ ! -b /dev/nbd0 ]; then echo "Failed to get /dev/nbd0, stopping." show_splash "ERROR: Failed to initialise netboot\\nhttps://postmarketos.org/netboot" - pmos_loop_forever + fail_halt_boot fi client_ip="${unudhcpd_client_ip:-172.16.42.2}"