postmarketos-initramfs: export logs on boot failure (MR 4646)

Currently, when postmarketOS fails to boot up, retrieving any
information necessary to root cause the issue is non-trivial, requiring
building a custom initramfs with debug-shell enabled and then manually
copying out data.

Let's improve the situation by exporting logs automatically on boot
failure. This is safer than just exposing a root shell but still
provides a whole lot of useful info that should make duplicating and
triaging issues much easier for developers.

This commit implements the functionality and includes some generally
useful data. We then generate a README and expose files for each command/log
as well as an archive that can be easily attached to a GitLab issue.

To help with triaging, also record the version of the postmarketos-initramfs
package and hash the init.sh and init_functions.sh files.

For testing purposes, you can trigger the log recovery mode on-time by
creating an empty file named ".pmos_export_logs" in the /boot partition.

Signed-off-by: Caleb Connolly <caleb@connolly.tech>

main/postmarketos-initramfs/APKBUILD

@@ -1,7 +1,7 @@
 # Maintainer: Oliver Smith <ollieparanoid@postmarketos.org>
 # Co-Maintainer: Clayton Craft <clayton@craftyguy.net>
 pkgname=postmarketos-initramfs
-pkgver=2.3.0
+pkgver=2.4.0
 pkgrel=0
 pkgdesc="Base files for the postmarketOS initramfs / initramfs-extra"
 url="https://postmarketos.org"
@@ -39,6 +39,14 @@ source="
 arch="noarch"
 license="GPL-2.0-or-later"
 
+build() {
+	mkdir -p "$builddir"
+
+	# Replace <<INITRAMFS_PKG_VERSION>> with the actual version
+	sed "s|<<INITRAMFS_PKG_VERSION>>|$pkgver-r$pkgrel|" \
+		"$srcdir/init.sh" > "$builddir/init.sh"
+}
+
 package() {
 	install -Dm644 "$srcdir/unudhcpd.conf" \
 		"$pkgdir/etc/unudhcpd.conf"
@@ -46,7 +54,7 @@ package() {
 	install -Dm644 "$srcdir/init_functions.sh" \
 		"$pkgdir/usr/share/initramfs/init_functions.sh"
 
-	install -Dm755 "$srcdir/init.sh" \
+	install -Dm755 "$builddir/init.sh" \
 		"$pkgdir/usr/share/initramfs/init.sh"
 
 	install -Dm644 "$srcdir"/00-initramfs-base.dirs \
@@ -79,8 +87,8 @@ sha512sums="
 5b364300f31c91fd0591eb0715f67cbf5383f45246a5fb9f34b79f7cb2e3b15768b2130e5f32f816cc169950f988c1beabc879ba31645c58ce131a288dbc071d  00-initramfs-base.dirs
 ab41b45b0613f25a61114ed8c8b92bc53c60838f6e2e0ba18c76e5369b2984e6023a0661887692673aca3f647f268c468a468f6b1ac424cfee609017a89481dd  00-initramfs-base.files
 8a4adad3785af474b36a09a05f6a3b2c4b4f43aac331a53b903abfa51ea12be1e3d1d807b7a6e66a1346815f3b0044daf8cd62e21e2dc75d2db13ee265a72985  00-initramfs-extra-base.files
-701b6252a536dfc7bddef0f5c3ccb16ad680f70efbad872b74d61ef24c98edcf623b19b5e1a2973bc3fa66ca009a515d7ba3563f0d417306a0a2b5680bee7efd  init.sh
+4bfcb41b2d5ccd577a6203a9479e5bdf056064d20f02f9120f1a074c7c280c0231cf380ed6be7c80733ed4ae101385c14f123a8a41780bbb7d0851387cd9c613  init.sh
-74cfd8e30811bcf3bb075bcc4db0b59c4f63ba83552cfe371132dacd4c41d73c04c65d309609578369243e9b969feba56b4bdd478eb5e7ebff7099e84f7c0f01  init_functions.sh
+58ee2b36b858c9c70fabbcb9a12a754afc97e6dfe09ddb7a2ec02c6fb593cc40677ae7049ba5ffda7797cba20e4d35f0d320c5fb4d3072cdbdf172cce6665bb0  init_functions.sh
 ba3275a9af788c7c782322a22a0f144d5e50e3498ea6886486a29331f23ae89cd32d500a3635cfa7cab369afba92edc18aeca64ccbf0cd589061cce23d15b46c  unudhcpd.conf
 675e7d5bee39b2df7d322117f8dcaccc274d61beaf4d50ead19bbf2109446d64b1c0aa0c5b4f9846eb6c1c403418f28f6364eff4537ba41120fbfcbc484b7da7  mdev.conf
 "

main/postmarketos-initramfs/init.sh

@@ -14,6 +14,9 @@ IN_CI="false"
 # provide a default for os-release's VERSION in case the file doesn't exist
 VERSION="${VERSION:-unknown}"
 
+# This is set during packaging and is used when triaging bug reports
+INITRAMFS_PKG_VERSION="<<INITRAMFS_PKG_VERSION>>"
+
 export PATH=/usr/bin:/bin:/usr/sbin:/sbin
 /bin/busybox --install -s
 /bin/busybox-extras --install -s
@@ -40,7 +43,7 @@ run_hooks /hooks
 if [ "$IN_CI" = "true" ]; then
 	echo "PMOS: CI tests done, disabling console and looping forever"
 	dmesg -n 1
-	loop_forever
+	fail_halt_boot
 fi
 
 # Always run dhcp daemon/usb networking for now (later this should only
@@ -54,6 +57,19 @@ extract_initramfs_extra /boot/initramfs-extra
 setup_udev
 run_hooks /hooks-extra
 
+# For testing the mass storage gadget log export function. We use a flag
+# file on /boot so that we can test it on all devices as modifying the
+# kernel cmdline is not always possible.
+if [ -e /boot/.pmos_export_logs ]; then
+	echo "PMOS: Exporting logs via mass storage gadget"
+	show_splash "Exporting boot logs..."
+	# Delete the flag so we don't soft-brick the device by always booting
+	# to the log export mode.
+	mount -o remount,rw /boot
+	rm -f /boot/.pmos_export_logs
+	fail_halt_boot
+fi
+
 wait_root_partition
 delete_old_install_partition
 resize_root_partition
@@ -77,4 +93,4 @@ exec switch_root /sysroot "$init"
 echo "ERROR: switch_root failed!"
 echo "Looping forever. Install and use the debug-shell hook to debug this."
 echo "For more information, see <https://postmarketos.org/debug-shell>"
-loop_forever
+fail_halt_boot

main/postmarketos-initramfs/init_functions.sh

@@ -304,7 +304,7 @@ mount_boot_partition() {
 	if [ -z "$partition" ]; then
 		echo "ERROR: boot partition not found!"
 		show_splash "ERROR: Boot partition not found\\nhttps://postmarketos.org/troubleshooting"
-		loop_forever
+		fail_halt_boot
 	fi
 
 	if [ "$2" = "rw" ]; then
@@ -341,7 +341,7 @@ extract_initramfs_extra() {
 	if [ ! -e "$initramfs_extra" ]; then
 		echo "ERROR: initramfs-extra not found!"
 		show_splash "ERROR: initramfs-extra not found\\nhttps://postmarketos.org/troubleshooting"
-		loop_forever
+		fail_halt_boot
 	fi
 	echo "Extract $initramfs_extra"
 	# uncompressed:
@@ -513,7 +513,7 @@ mount_root_partition() {
 	if ! { [ "$type" = "ext4" ] || [ "$type" = "f2fs" ] || [ "$type" = "btrfs" ]; } then
 		echo "ERROR: Detected unsupported '$type' filesystem ($partition)."
 		show_splash "ERROR: unsupported '$type' filesystem ($partition)\\nhttps://postmarketos.org/troubleshooting"
-		loop_forever
+		fail_halt_boot
 	fi
 
 	if ! modprobe "$type"; then
@@ -522,13 +522,13 @@ mount_root_partition() {
 	if ! mount -t "$type" -o ro"$rootfsopts" "$partition" /sysroot; then
 		echo "ERROR: unable to mount root partition!"
 		show_splash "ERROR: unable to mount root partition\\nhttps://postmarketos.org/troubleshooting"
-		loop_forever
+		fail_halt_boot
 	fi
 
 	if ! [ -e /sysroot/usr ]; then
 		echo "ERROR: root partition appeared to mount but does not contain a root filesystem!"
 		show_splash "ERROR: root partition does not contain a root filesystem\\nhttps://postmarketos.org/troubleshooting"
-		loop_forever
+		fail_halt_boot
 	fi
 }
 
@@ -584,9 +584,11 @@ setup_usb_configfs_udc() {
 	echo "$_udc_dev" > /config/usb_gadget/g1/UDC || echo "  Couldn't write new UDC"
 }
 
+# $1: if set, skip writing to the UDC
 setup_usb_network_configfs() {
 	# See: https://www.kernel.org/doc/Documentation/usb/gadget_configfs.txt
 	CONFIGFS=/config/usb_gadget
+	local skip_udc="$1"
 
 	if ! [ -e "$CONFIGFS" ]; then
 		echo "  /config/usb_gadget does not exist, skipping configfs usb gadget"
@@ -638,7 +640,11 @@ setup_usb_network_configfs() {
 	ln -s $CONFIGFS/g1/functions/"$usb_network_function" $CONFIGFS/g1/configs/c.1 \
 		|| echo "  Couldn't symlink $usb_network_function"
 
+	# If an argument was supplied then skip writing to the UDC (only used for mass storage
+	# log recovery)
+	if [ -z "$skip_udc" ]; then
 		setup_usb_configfs_udc
+	fi
 }
 
 setup_usb_network() {
@@ -771,7 +777,98 @@ setup_bootchart2() {
 	fi
 }
 
-loop_forever() {
+mkhash() {
+	sha256sum "$1" | cut -d " " -f 1
+}
+
+# Create a small disk image and copy logs to it so they can be exposed via mass storage
+create_logs_disk() {
+	local loop_dev="$1"
+	local upload_file=""
+	echo "Creating logs disk"
+
+	fallocate -l 10M /tmp/logs.img
+	# The log device used is assumed to be $loop_dev
+	losetup -f /tmp/logs.img
+	mkfs.vfat -n "PMOS_LOGS" "$loop_dev"
+	mkdir -p /tmp/logs
+	mount "$loop_dev" /tmp/logs
+
+	# Copy logs
+	cp /pmOS_init.log /tmp/logs/pmOS_init.txt
+	dmesg > /tmp/logs/dmesg.txt
+	blkid > /tmp/logs/blkid.txt
+	cat /proc/cmdline > /tmp/logs/cmdline.txt
+	cat /proc/partitions > /tmp/logs/partitions.txt
+	# Include FDT if it exists
+	[ -e /sys/firmware/fdt ] && cp /sys/firmware/fdt /tmp/logs/fdt.dtb
+
+	# Additional info about the initramfs
+	{
+		echo "initramfs-version: $INITRAMFS_PKG_VERSION"
+		# Take hashes of the initramfs files so we can be sure they weren't modified inadvertantly
+		echo "init-hash: $(mkhash /init)"
+		echo "init-functions-hash: $(mkhash /init_functions.sh)"
+	} >> /tmp/logs/_info
+
+	# Create a tar file with all the logs. We don't include the date because on many devices
+	# (especially Qualcomm) the RTC is likely wrong.
+	upload_file="${deviceinfo_codename}-${VERSION}-$(uname -r).tar.gz"
+	# Done in a subshell to not change the working directory of init
+	(cd /tmp/logs || ( echo "Couldn't cd to /tmp/logs"; return ); tar -cv ./* | gzip -6 -c > "/tmp/$upload_file")
+	mv "/tmp/$upload_file" /tmp/logs/
+
+	# Create a README with instructions on how to report an issue
+	cat > /tmp/logs/README.txt <<-EOF
+	Something went wrong and your device did not boot properly. If this was unexpected
+	then please open a new issue by visiting
+
+	https://gitlab.com/postmarketOS/pmaports/-/issues/new
+
+	and attach the following file by dragging it onto the page:
+
+	* $upload_file
+
+	You are running postmarketOS $VERSION on kernel $(uname -r).
+	EOF
+
+	# Unmount
+	umount /tmp/logs
+}
+
+# Make logs available via mass storage gadget
+export_logs() {
+	local loop_dev=""
+	usb_mass_storage_function="mass_storage.0"
+	active_udc="$(cat /config/usb_gadget/g1/UDC)"
+
+	loop_dev="$(losetup -f)"
+
+	create_logs_disk "$loop_dev"
+
+	echo "Making logs available via mass storage"
+
+	# Set up network gadget if not already done
+	if [ -z "$active_udc" ]; then
+		setup_usb_network_configfs "skip_udc"
+	else
+		# Unset UDC
+		echo "" > /config/usb_gadget/g1/UDC
+	fi
+
+	mkdir "$CONFIGFS"/g1/functions/"$usb_mass_storage_function" || return
+
+	echo "$loop_dev" > "$CONFIGFS"/g1/functions/"$usb_mass_storage_function"/lun.0/file
+
+	ln -s "$CONFIGFS"/g1/functions/"$usb_mass_storage_function" \
+		"$CONFIGFS"/g1/configs/c.1 || return
+
+	setup_usb_configfs_udc
+}
+
+fail_halt_boot() {
+	export_logs
+	echo "Looping forever"
 	while true; do
 		sleep 1
 	done

main/postmarketos-mkinitfs-hook-debug-shell/20-debug-shell.sh

@@ -15,7 +15,7 @@ echo "Create 'pmos_continue_boot' script"
 	#Disable any active usb mass storage
 	echo "if [ -d /config/usb_gadget/g1/functions/mass_storage.0 ]; then setup_usb_storage; fi"
 	echo "pkill -9 -f pmos_shell"
-	echo "pkill -f pmos_loop_forever"
+	echo "pkill -f pmos_fail_halt_boot"
 	echo "pkill -f telnetd.*:${TELNET_PORT}"
 } >/usr/bin/pmos_continue_boot
 chmod +x /usr/bin/pmos_continue_boot
@@ -27,13 +27,13 @@ echo "Create 'pmos_shell' script"
 } >/usr/bin/pmos_shell
 chmod +x /usr/bin/pmos_shell
 
-echo "Create 'pmos_loop_forever' script"
+echo "Create 'pmos_fail_halt_boot' script"
 {
 	echo "#!/bin/sh"
 	echo '. /init_functions.sh'
-	echo "loop_forever"
+	echo "fail_halt_boot"
-} >/usr/bin/pmos_loop_forever
+} >/usr/bin/pmos_fail_halt_boot
-chmod +x /usr/bin/pmos_loop_forever
+chmod +x /usr/bin/pmos_fail_halt_boot
 
 echo "Start the telnet daemon"
 {

main/postmarketos-mkinitfs-hook-debug-shell/APKBUILD

@@ -1,5 +1,5 @@
 pkgname=postmarketos-mkinitfs-hook-debug-shell
-pkgver=0.5.0
+pkgver=0.6.0
 pkgrel=0
 pkgdesc="Root shell in the initramfs (security hole, for debugging only)"
 url="https://postmarketos.org"
@@ -19,7 +19,7 @@ package() {
 }
 
 sha512sums="
-0750a33f3b68fd6f751bdcf3c59656ec8c8af60e13c25b13394937134d4cbeaacd237b5d92e2a978319f705781f39712c71673367cec52b584f666af50b6f952  20-debug-shell.sh
+add95b3fa64b804fd386298406fbc55489e3e122d70f40e45627a450d299e148a346a4a147c2b28e0110ecef14e9e0b62ac75a0bd2d39295c2407ebb5e93708c  20-debug-shell.sh
 a739bc47d905d189edb26d9ebfd062023720fefdaab27207471c16d53a9c12ea8b81092a1047d8f2300e42ba500bdf6c5a3343aca55aab5bf8e84d68eb5680ab  20-debug-shell.files
 75d485c2e9f352cfd717b7a92753a9dfc4a72526a44bcbb784eacb4ef9011072b3ffa1c42a317c0940598cc076fb6c61676c440e5b188378b19ca08d882c1338  setup_usb_storage.sh
 "

main/postmarketos-mkinitfs-hook-maximum-attention/00-maximum-attention.sh

@@ -53,4 +53,4 @@ vibrate_loop() {
 blink_leds $(find_leds) &
 vibrate_loop $(find_vibrator) &
 
-loop_forever
+fail_halt_boot

main/postmarketos-mkinitfs-hook-maximum-attention/APKBUILD

@@ -1,6 +1,6 @@
 pkgname=postmarketos-mkinitfs-hook-maximum-attention
-pkgver=0.1.0
+pkgver=0.2.0
-pkgrel=1
+pkgrel=0
 pkgdesc="Script to activate all user-visible outputs from the initramfs (to confirm working kernel, for debugging only)"
 url="https://postmarketos.org"
 depends="postmarketos-mkinitfs"
@@ -12,4 +12,6 @@ package() {
 	install -Dm644 "$srcdir"/00-maximum-attention.sh \
 		-t "$pkgdir"/usr/share/mkinitfs/hooks/
 }
-sha512sums="c29cc41649c284c0ec8201715cd9452f838801a22ea6c2a312ea87a58c61c1caa05a3b1ff33ce4a4d680a95e79fb10108308b72173f1fa97bac5cefde7ab30ee  00-maximum-attention.sh"
+sha512sums="
+ca3ef48e76e21fdb67fdf7477fb4739e329ce6ea04a52ac14e9365adb92058b21e08fcbc4020519c61ba20cd3c91d254118ece3d1818cb46f86e86f5ffa1b90a  00-maximum-attention.sh
+"

main/postmarketos-mkinitfs-hook-netboot/APKBUILD

@@ -1,6 +1,6 @@
 # Maintainer: Mark <clashclanacc2602@gmail.com>
 pkgname=postmarketos-mkinitfs-hook-netboot
-pkgver=4.0.3
+pkgver=4.1.3
 pkgrel=0
 pkgdesc="Netboot postmarketOS using nbd feature"
 url="https://postmarketos.org"
@@ -21,6 +21,6 @@ package() {
 }
 
 sha512sums="
-039b6ae08ffb2e24573a5659fef8ff14f70219e06fe6dd6c5ad459f3b54883e17166060813762cdc71b7c774f450956158871fd64c70330d0a661a8cc94666e7  netboot.sh
+c681d0c39eecd9fe5d284e07f9c6f79e41085628abe1c645b8ea3cc4e621fd603fca9db4a555fa689a3268925cb6d23c12baa98986baed452375acd1e3fe26b1  netboot.sh
 3c9c812a6cd25640e6cf4d722cff6c6356d5666ec6009b4c65f89e62211cc406d2cd09e0fdda6b1c1906ee32c3854988ab701cee8811208e3ef666f09e638568  netboot.modules
 "

main/postmarketos-mkinitfs-hook-netboot/netboot.sh

@@ -17,7 +17,7 @@ modprobe nbd
 if [ ! -b /dev/nbd0 ]; then
 	echo "Failed to get /dev/nbd0, stopping."
 	show_splash "ERROR: Failed to initialise netboot\\nhttps://postmarketos.org/netboot"
-	pmos_loop_forever
+	fail_halt_boot
 fi
 
 client_ip="${unudhcpd_client_ip:-172.16.42.2}"