postmarketos-initramfs: export logs on boot failure (MR 4646)

Currently, when postmarketOS fails to boot up, retrieving any
information necessary to root cause the issue is non-trivial, requiring
building a custom initramfs with debug-shell enabled and then manually
copying out data.

Let's improve the situation by exporting logs automatically on boot
failure. This is safer than just exposing a root shell but still
provides a whole lot of useful info that should make duplicating and
triaging issues much easier for developers.

This commit implements the functionality and includes some generally
useful data. We then generate a README and expose files for each command/log
as well as an archive that can be easily attached to a GitLab issue.

To help with triaging, also record the version of the postmarketos-initramfs
package and hash the init.sh and init_functions.sh files.

For testing purposes, you can trigger the log recovery mode on-time by
creating an empty file named ".pmos_export_logs" in the /boot partition.

Signed-off-by: Caleb Connolly <caleb@connolly.tech>
This commit is contained in:
Caleb Connolly 2023-12-18 22:48:14 +00:00
parent 241ade087f
commit 57744b984e
No known key found for this signature in database
GPG key ID: 7930459FB9303217
9 changed files with 150 additions and 27 deletions

View file

@ -1,7 +1,7 @@
# Maintainer: Oliver Smith <ollieparanoid@postmarketos.org> # Maintainer: Oliver Smith <ollieparanoid@postmarketos.org>
# Co-Maintainer: Clayton Craft <clayton@craftyguy.net> # Co-Maintainer: Clayton Craft <clayton@craftyguy.net>
pkgname=postmarketos-initramfs pkgname=postmarketos-initramfs
pkgver=2.3.0 pkgver=2.4.0
pkgrel=0 pkgrel=0
pkgdesc="Base files for the postmarketOS initramfs / initramfs-extra" pkgdesc="Base files for the postmarketOS initramfs / initramfs-extra"
url="https://postmarketos.org" url="https://postmarketos.org"
@ -39,6 +39,14 @@ source="
arch="noarch" arch="noarch"
license="GPL-2.0-or-later" license="GPL-2.0-or-later"
build() {
mkdir -p "$builddir"
# Replace <<INITRAMFS_PKG_VERSION>> with the actual version
sed "s|<<INITRAMFS_PKG_VERSION>>|$pkgver-r$pkgrel|" \
"$srcdir/init.sh" > "$builddir/init.sh"
}
package() { package() {
install -Dm644 "$srcdir/unudhcpd.conf" \ install -Dm644 "$srcdir/unudhcpd.conf" \
"$pkgdir/etc/unudhcpd.conf" "$pkgdir/etc/unudhcpd.conf"
@ -46,7 +54,7 @@ package() {
install -Dm644 "$srcdir/init_functions.sh" \ install -Dm644 "$srcdir/init_functions.sh" \
"$pkgdir/usr/share/initramfs/init_functions.sh" "$pkgdir/usr/share/initramfs/init_functions.sh"
install -Dm755 "$srcdir/init.sh" \ install -Dm755 "$builddir/init.sh" \
"$pkgdir/usr/share/initramfs/init.sh" "$pkgdir/usr/share/initramfs/init.sh"
install -Dm644 "$srcdir"/00-initramfs-base.dirs \ install -Dm644 "$srcdir"/00-initramfs-base.dirs \
@ -79,8 +87,8 @@ sha512sums="
5b364300f31c91fd0591eb0715f67cbf5383f45246a5fb9f34b79f7cb2e3b15768b2130e5f32f816cc169950f988c1beabc879ba31645c58ce131a288dbc071d 00-initramfs-base.dirs 5b364300f31c91fd0591eb0715f67cbf5383f45246a5fb9f34b79f7cb2e3b15768b2130e5f32f816cc169950f988c1beabc879ba31645c58ce131a288dbc071d 00-initramfs-base.dirs
ab41b45b0613f25a61114ed8c8b92bc53c60838f6e2e0ba18c76e5369b2984e6023a0661887692673aca3f647f268c468a468f6b1ac424cfee609017a89481dd 00-initramfs-base.files ab41b45b0613f25a61114ed8c8b92bc53c60838f6e2e0ba18c76e5369b2984e6023a0661887692673aca3f647f268c468a468f6b1ac424cfee609017a89481dd 00-initramfs-base.files
8a4adad3785af474b36a09a05f6a3b2c4b4f43aac331a53b903abfa51ea12be1e3d1d807b7a6e66a1346815f3b0044daf8cd62e21e2dc75d2db13ee265a72985 00-initramfs-extra-base.files 8a4adad3785af474b36a09a05f6a3b2c4b4f43aac331a53b903abfa51ea12be1e3d1d807b7a6e66a1346815f3b0044daf8cd62e21e2dc75d2db13ee265a72985 00-initramfs-extra-base.files
701b6252a536dfc7bddef0f5c3ccb16ad680f70efbad872b74d61ef24c98edcf623b19b5e1a2973bc3fa66ca009a515d7ba3563f0d417306a0a2b5680bee7efd init.sh 4bfcb41b2d5ccd577a6203a9479e5bdf056064d20f02f9120f1a074c7c280c0231cf380ed6be7c80733ed4ae101385c14f123a8a41780bbb7d0851387cd9c613 init.sh
74cfd8e30811bcf3bb075bcc4db0b59c4f63ba83552cfe371132dacd4c41d73c04c65d309609578369243e9b969feba56b4bdd478eb5e7ebff7099e84f7c0f01 init_functions.sh 58ee2b36b858c9c70fabbcb9a12a754afc97e6dfe09ddb7a2ec02c6fb593cc40677ae7049ba5ffda7797cba20e4d35f0d320c5fb4d3072cdbdf172cce6665bb0 init_functions.sh
ba3275a9af788c7c782322a22a0f144d5e50e3498ea6886486a29331f23ae89cd32d500a3635cfa7cab369afba92edc18aeca64ccbf0cd589061cce23d15b46c unudhcpd.conf ba3275a9af788c7c782322a22a0f144d5e50e3498ea6886486a29331f23ae89cd32d500a3635cfa7cab369afba92edc18aeca64ccbf0cd589061cce23d15b46c unudhcpd.conf
675e7d5bee39b2df7d322117f8dcaccc274d61beaf4d50ead19bbf2109446d64b1c0aa0c5b4f9846eb6c1c403418f28f6364eff4537ba41120fbfcbc484b7da7 mdev.conf 675e7d5bee39b2df7d322117f8dcaccc274d61beaf4d50ead19bbf2109446d64b1c0aa0c5b4f9846eb6c1c403418f28f6364eff4537ba41120fbfcbc484b7da7 mdev.conf
" "

View file

@ -14,6 +14,9 @@ IN_CI="false"
# provide a default for os-release's VERSION in case the file doesn't exist # provide a default for os-release's VERSION in case the file doesn't exist
VERSION="${VERSION:-unknown}" VERSION="${VERSION:-unknown}"
# This is set during packaging and is used when triaging bug reports
INITRAMFS_PKG_VERSION="<<INITRAMFS_PKG_VERSION>>"
export PATH=/usr/bin:/bin:/usr/sbin:/sbin export PATH=/usr/bin:/bin:/usr/sbin:/sbin
/bin/busybox --install -s /bin/busybox --install -s
/bin/busybox-extras --install -s /bin/busybox-extras --install -s
@ -40,7 +43,7 @@ run_hooks /hooks
if [ "$IN_CI" = "true" ]; then if [ "$IN_CI" = "true" ]; then
echo "PMOS: CI tests done, disabling console and looping forever" echo "PMOS: CI tests done, disabling console and looping forever"
dmesg -n 1 dmesg -n 1
loop_forever fail_halt_boot
fi fi
# Always run dhcp daemon/usb networking for now (later this should only # Always run dhcp daemon/usb networking for now (later this should only
@ -54,6 +57,19 @@ extract_initramfs_extra /boot/initramfs-extra
setup_udev setup_udev
run_hooks /hooks-extra run_hooks /hooks-extra
# For testing the mass storage gadget log export function. We use a flag
# file on /boot so that we can test it on all devices as modifying the
# kernel cmdline is not always possible.
if [ -e /boot/.pmos_export_logs ]; then
echo "PMOS: Exporting logs via mass storage gadget"
show_splash "Exporting boot logs..."
# Delete the flag so we don't soft-brick the device by always booting
# to the log export mode.
mount -o remount,rw /boot
rm -f /boot/.pmos_export_logs
fail_halt_boot
fi
wait_root_partition wait_root_partition
delete_old_install_partition delete_old_install_partition
resize_root_partition resize_root_partition
@ -77,4 +93,4 @@ exec switch_root /sysroot "$init"
echo "ERROR: switch_root failed!" echo "ERROR: switch_root failed!"
echo "Looping forever. Install and use the debug-shell hook to debug this." echo "Looping forever. Install and use the debug-shell hook to debug this."
echo "For more information, see <https://postmarketos.org/debug-shell>" echo "For more information, see <https://postmarketos.org/debug-shell>"
loop_forever fail_halt_boot

View file

@ -304,7 +304,7 @@ mount_boot_partition() {
if [ -z "$partition" ]; then if [ -z "$partition" ]; then
echo "ERROR: boot partition not found!" echo "ERROR: boot partition not found!"
show_splash "ERROR: Boot partition not found\\nhttps://postmarketos.org/troubleshooting" show_splash "ERROR: Boot partition not found\\nhttps://postmarketos.org/troubleshooting"
loop_forever fail_halt_boot
fi fi
if [ "$2" = "rw" ]; then if [ "$2" = "rw" ]; then
@ -341,7 +341,7 @@ extract_initramfs_extra() {
if [ ! -e "$initramfs_extra" ]; then if [ ! -e "$initramfs_extra" ]; then
echo "ERROR: initramfs-extra not found!" echo "ERROR: initramfs-extra not found!"
show_splash "ERROR: initramfs-extra not found\\nhttps://postmarketos.org/troubleshooting" show_splash "ERROR: initramfs-extra not found\\nhttps://postmarketos.org/troubleshooting"
loop_forever fail_halt_boot
fi fi
echo "Extract $initramfs_extra" echo "Extract $initramfs_extra"
# uncompressed: # uncompressed:
@ -513,7 +513,7 @@ mount_root_partition() {
if ! { [ "$type" = "ext4" ] || [ "$type" = "f2fs" ] || [ "$type" = "btrfs" ]; } then if ! { [ "$type" = "ext4" ] || [ "$type" = "f2fs" ] || [ "$type" = "btrfs" ]; } then
echo "ERROR: Detected unsupported '$type' filesystem ($partition)." echo "ERROR: Detected unsupported '$type' filesystem ($partition)."
show_splash "ERROR: unsupported '$type' filesystem ($partition)\\nhttps://postmarketos.org/troubleshooting" show_splash "ERROR: unsupported '$type' filesystem ($partition)\\nhttps://postmarketos.org/troubleshooting"
loop_forever fail_halt_boot
fi fi
if ! modprobe "$type"; then if ! modprobe "$type"; then
@ -522,13 +522,13 @@ mount_root_partition() {
if ! mount -t "$type" -o ro"$rootfsopts" "$partition" /sysroot; then if ! mount -t "$type" -o ro"$rootfsopts" "$partition" /sysroot; then
echo "ERROR: unable to mount root partition!" echo "ERROR: unable to mount root partition!"
show_splash "ERROR: unable to mount root partition\\nhttps://postmarketos.org/troubleshooting" show_splash "ERROR: unable to mount root partition\\nhttps://postmarketos.org/troubleshooting"
loop_forever fail_halt_boot
fi fi
if ! [ -e /sysroot/usr ]; then if ! [ -e /sysroot/usr ]; then
echo "ERROR: root partition appeared to mount but does not contain a root filesystem!" echo "ERROR: root partition appeared to mount but does not contain a root filesystem!"
show_splash "ERROR: root partition does not contain a root filesystem\\nhttps://postmarketos.org/troubleshooting" show_splash "ERROR: root partition does not contain a root filesystem\\nhttps://postmarketos.org/troubleshooting"
loop_forever fail_halt_boot
fi fi
} }
@ -584,9 +584,11 @@ setup_usb_configfs_udc() {
echo "$_udc_dev" > /config/usb_gadget/g1/UDC || echo " Couldn't write new UDC" echo "$_udc_dev" > /config/usb_gadget/g1/UDC || echo " Couldn't write new UDC"
} }
# $1: if set, skip writing to the UDC
setup_usb_network_configfs() { setup_usb_network_configfs() {
# See: https://www.kernel.org/doc/Documentation/usb/gadget_configfs.txt # See: https://www.kernel.org/doc/Documentation/usb/gadget_configfs.txt
CONFIGFS=/config/usb_gadget CONFIGFS=/config/usb_gadget
local skip_udc="$1"
if ! [ -e "$CONFIGFS" ]; then if ! [ -e "$CONFIGFS" ]; then
echo " /config/usb_gadget does not exist, skipping configfs usb gadget" echo " /config/usb_gadget does not exist, skipping configfs usb gadget"
@ -638,7 +640,11 @@ setup_usb_network_configfs() {
ln -s $CONFIGFS/g1/functions/"$usb_network_function" $CONFIGFS/g1/configs/c.1 \ ln -s $CONFIGFS/g1/functions/"$usb_network_function" $CONFIGFS/g1/configs/c.1 \
|| echo " Couldn't symlink $usb_network_function" || echo " Couldn't symlink $usb_network_function"
setup_usb_configfs_udc # If an argument was supplied then skip writing to the UDC (only used for mass storage
# log recovery)
if [ -z "$skip_udc" ]; then
setup_usb_configfs_udc
fi
} }
setup_usb_network() { setup_usb_network() {
@ -771,7 +777,98 @@ setup_bootchart2() {
fi fi
} }
loop_forever() { mkhash() {
sha256sum "$1" | cut -d " " -f 1
}
# Create a small disk image and copy logs to it so they can be exposed via mass storage
create_logs_disk() {
local loop_dev="$1"
local upload_file=""
echo "Creating logs disk"
fallocate -l 10M /tmp/logs.img
# The log device used is assumed to be $loop_dev
losetup -f /tmp/logs.img
mkfs.vfat -n "PMOS_LOGS" "$loop_dev"
mkdir -p /tmp/logs
mount "$loop_dev" /tmp/logs
# Copy logs
cp /pmOS_init.log /tmp/logs/pmOS_init.txt
dmesg > /tmp/logs/dmesg.txt
blkid > /tmp/logs/blkid.txt
cat /proc/cmdline > /tmp/logs/cmdline.txt
cat /proc/partitions > /tmp/logs/partitions.txt
# Include FDT if it exists
[ -e /sys/firmware/fdt ] && cp /sys/firmware/fdt /tmp/logs/fdt.dtb
# Additional info about the initramfs
{
echo "initramfs-version: $INITRAMFS_PKG_VERSION"
# Take hashes of the initramfs files so we can be sure they weren't modified inadvertantly
echo "init-hash: $(mkhash /init)"
echo "init-functions-hash: $(mkhash /init_functions.sh)"
} >> /tmp/logs/_info
# Create a tar file with all the logs. We don't include the date because on many devices
# (especially Qualcomm) the RTC is likely wrong.
upload_file="${deviceinfo_codename}-${VERSION}-$(uname -r).tar.gz"
# Done in a subshell to not change the working directory of init
(cd /tmp/logs || ( echo "Couldn't cd to /tmp/logs"; return ); tar -cv ./* | gzip -6 -c > "/tmp/$upload_file")
mv "/tmp/$upload_file" /tmp/logs/
# Create a README with instructions on how to report an issue
cat > /tmp/logs/README.txt <<-EOF
Something went wrong and your device did not boot properly. If this was unexpected
then please open a new issue by visiting
https://gitlab.com/postmarketOS/pmaports/-/issues/new
and attach the following file by dragging it onto the page:
* $upload_file
You are running postmarketOS $VERSION on kernel $(uname -r).
EOF
# Unmount
umount /tmp/logs
}
# Make logs available via mass storage gadget
export_logs() {
local loop_dev=""
usb_mass_storage_function="mass_storage.0"
active_udc="$(cat /config/usb_gadget/g1/UDC)"
loop_dev="$(losetup -f)"
create_logs_disk "$loop_dev"
echo "Making logs available via mass storage"
# Set up network gadget if not already done
if [ -z "$active_udc" ]; then
setup_usb_network_configfs "skip_udc"
else
# Unset UDC
echo "" > /config/usb_gadget/g1/UDC
fi
mkdir "$CONFIGFS"/g1/functions/"$usb_mass_storage_function" || return
echo "$loop_dev" > "$CONFIGFS"/g1/functions/"$usb_mass_storage_function"/lun.0/file
ln -s "$CONFIGFS"/g1/functions/"$usb_mass_storage_function" \
"$CONFIGFS"/g1/configs/c.1 || return
setup_usb_configfs_udc
}
fail_halt_boot() {
export_logs
echo "Looping forever"
while true; do while true; do
sleep 1 sleep 1
done done

View file

@ -15,7 +15,7 @@ echo "Create 'pmos_continue_boot' script"
#Disable any active usb mass storage #Disable any active usb mass storage
echo "if [ -d /config/usb_gadget/g1/functions/mass_storage.0 ]; then setup_usb_storage; fi" echo "if [ -d /config/usb_gadget/g1/functions/mass_storage.0 ]; then setup_usb_storage; fi"
echo "pkill -9 -f pmos_shell" echo "pkill -9 -f pmos_shell"
echo "pkill -f pmos_loop_forever" echo "pkill -f pmos_fail_halt_boot"
echo "pkill -f telnetd.*:${TELNET_PORT}" echo "pkill -f telnetd.*:${TELNET_PORT}"
} >/usr/bin/pmos_continue_boot } >/usr/bin/pmos_continue_boot
chmod +x /usr/bin/pmos_continue_boot chmod +x /usr/bin/pmos_continue_boot
@ -27,13 +27,13 @@ echo "Create 'pmos_shell' script"
} >/usr/bin/pmos_shell } >/usr/bin/pmos_shell
chmod +x /usr/bin/pmos_shell chmod +x /usr/bin/pmos_shell
echo "Create 'pmos_loop_forever' script" echo "Create 'pmos_fail_halt_boot' script"
{ {
echo "#!/bin/sh" echo "#!/bin/sh"
echo '. /init_functions.sh' echo '. /init_functions.sh'
echo "loop_forever" echo "fail_halt_boot"
} >/usr/bin/pmos_loop_forever } >/usr/bin/pmos_fail_halt_boot
chmod +x /usr/bin/pmos_loop_forever chmod +x /usr/bin/pmos_fail_halt_boot
echo "Start the telnet daemon" echo "Start the telnet daemon"
{ {

View file

@ -1,5 +1,5 @@
pkgname=postmarketos-mkinitfs-hook-debug-shell pkgname=postmarketos-mkinitfs-hook-debug-shell
pkgver=0.5.0 pkgver=0.6.0
pkgrel=0 pkgrel=0
pkgdesc="Root shell in the initramfs (security hole, for debugging only)" pkgdesc="Root shell in the initramfs (security hole, for debugging only)"
url="https://postmarketos.org" url="https://postmarketos.org"
@ -19,7 +19,7 @@ package() {
} }
sha512sums=" sha512sums="
0750a33f3b68fd6f751bdcf3c59656ec8c8af60e13c25b13394937134d4cbeaacd237b5d92e2a978319f705781f39712c71673367cec52b584f666af50b6f952 20-debug-shell.sh add95b3fa64b804fd386298406fbc55489e3e122d70f40e45627a450d299e148a346a4a147c2b28e0110ecef14e9e0b62ac75a0bd2d39295c2407ebb5e93708c 20-debug-shell.sh
a739bc47d905d189edb26d9ebfd062023720fefdaab27207471c16d53a9c12ea8b81092a1047d8f2300e42ba500bdf6c5a3343aca55aab5bf8e84d68eb5680ab 20-debug-shell.files a739bc47d905d189edb26d9ebfd062023720fefdaab27207471c16d53a9c12ea8b81092a1047d8f2300e42ba500bdf6c5a3343aca55aab5bf8e84d68eb5680ab 20-debug-shell.files
75d485c2e9f352cfd717b7a92753a9dfc4a72526a44bcbb784eacb4ef9011072b3ffa1c42a317c0940598cc076fb6c61676c440e5b188378b19ca08d882c1338 setup_usb_storage.sh 75d485c2e9f352cfd717b7a92753a9dfc4a72526a44bcbb784eacb4ef9011072b3ffa1c42a317c0940598cc076fb6c61676c440e5b188378b19ca08d882c1338 setup_usb_storage.sh
" "

View file

@ -53,4 +53,4 @@ vibrate_loop() {
blink_leds $(find_leds) & blink_leds $(find_leds) &
vibrate_loop $(find_vibrator) & vibrate_loop $(find_vibrator) &
loop_forever fail_halt_boot

View file

@ -1,6 +1,6 @@
pkgname=postmarketos-mkinitfs-hook-maximum-attention pkgname=postmarketos-mkinitfs-hook-maximum-attention
pkgver=0.1.0 pkgver=0.2.0
pkgrel=1 pkgrel=0
pkgdesc="Script to activate all user-visible outputs from the initramfs (to confirm working kernel, for debugging only)" pkgdesc="Script to activate all user-visible outputs from the initramfs (to confirm working kernel, for debugging only)"
url="https://postmarketos.org" url="https://postmarketos.org"
depends="postmarketos-mkinitfs" depends="postmarketos-mkinitfs"
@ -12,4 +12,6 @@ package() {
install -Dm644 "$srcdir"/00-maximum-attention.sh \ install -Dm644 "$srcdir"/00-maximum-attention.sh \
-t "$pkgdir"/usr/share/mkinitfs/hooks/ -t "$pkgdir"/usr/share/mkinitfs/hooks/
} }
sha512sums="c29cc41649c284c0ec8201715cd9452f838801a22ea6c2a312ea87a58c61c1caa05a3b1ff33ce4a4d680a95e79fb10108308b72173f1fa97bac5cefde7ab30ee 00-maximum-attention.sh" sha512sums="
ca3ef48e76e21fdb67fdf7477fb4739e329ce6ea04a52ac14e9365adb92058b21e08fcbc4020519c61ba20cd3c91d254118ece3d1818cb46f86e86f5ffa1b90a 00-maximum-attention.sh
"

View file

@ -1,6 +1,6 @@
# Maintainer: Mark <clashclanacc2602@gmail.com> # Maintainer: Mark <clashclanacc2602@gmail.com>
pkgname=postmarketos-mkinitfs-hook-netboot pkgname=postmarketos-mkinitfs-hook-netboot
pkgver=4.0.3 pkgver=4.1.3
pkgrel=0 pkgrel=0
pkgdesc="Netboot postmarketOS using nbd feature" pkgdesc="Netboot postmarketOS using nbd feature"
url="https://postmarketos.org" url="https://postmarketos.org"
@ -21,6 +21,6 @@ package() {
} }
sha512sums=" sha512sums="
039b6ae08ffb2e24573a5659fef8ff14f70219e06fe6dd6c5ad459f3b54883e17166060813762cdc71b7c774f450956158871fd64c70330d0a661a8cc94666e7 netboot.sh c681d0c39eecd9fe5d284e07f9c6f79e41085628abe1c645b8ea3cc4e621fd603fca9db4a555fa689a3268925cb6d23c12baa98986baed452375acd1e3fe26b1 netboot.sh
3c9c812a6cd25640e6cf4d722cff6c6356d5666ec6009b4c65f89e62211cc406d2cd09e0fdda6b1c1906ee32c3854988ab701cee8811208e3ef666f09e638568 netboot.modules 3c9c812a6cd25640e6cf4d722cff6c6356d5666ec6009b4c65f89e62211cc406d2cd09e0fdda6b1c1906ee32c3854988ab701cee8811208e3ef666f09e638568 netboot.modules
" "

View file

@ -17,7 +17,7 @@ modprobe nbd
if [ ! -b /dev/nbd0 ]; then if [ ! -b /dev/nbd0 ]; then
echo "Failed to get /dev/nbd0, stopping." echo "Failed to get /dev/nbd0, stopping."
show_splash "ERROR: Failed to initialise netboot\\nhttps://postmarketos.org/netboot" show_splash "ERROR: Failed to initialise netboot\\nhttps://postmarketos.org/netboot"
pmos_loop_forever fail_halt_boot
fi fi
client_ip="${unudhcpd_client_ip:-172.16.42.2}" client_ip="${unudhcpd_client_ip:-172.16.42.2}"