postmarketos-initramfs: export logs on boot failure (MR 4646)
Currently, when postmarketOS fails to boot up, retrieving any information necessary to root cause the issue is non-trivial, requiring building a custom initramfs with debug-shell enabled and then manually copying out data. Let's improve the situation by exporting logs automatically on boot failure. This is safer than just exposing a root shell but still provides a whole lot of useful info that should make duplicating and triaging issues much easier for developers. This commit implements the functionality and includes some generally useful data. We then generate a README and expose files for each command/log as well as an archive that can be easily attached to a GitLab issue. To help with triaging, also record the version of the postmarketos-initramfs package and hash the init.sh and init_functions.sh files. For testing purposes, you can trigger the log recovery mode on-time by creating an empty file named ".pmos_export_logs" in the /boot partition. Signed-off-by: Caleb Connolly <caleb@connolly.tech>
This commit is contained in:
Caleb Connolly
parent
241ade087f
commit
57744b984e
9 changed files with 150 additions and 27 deletions
|
|
@ -1,7 +1,7 @@
|
|||
# Maintainer: Oliver Smith <ollieparanoid@postmarketos.org>
|
||||
# Co-Maintainer: Clayton Craft <clayton@craftyguy.net>
|
||||
pkgname=postmarketos-initramfs
|
||||
pkgver=2.3.0
|
||||
pkgver=2.4.0
|
||||
pkgrel=0
|
||||
pkgdesc="Base files for the postmarketOS initramfs / initramfs-extra"
|
||||
url="https://postmarketos.org"
|
||||
|
|
@ -39,6 +39,14 @@ source="
|
|||
arch="noarch"
|
||||
license="GPL-2.0-or-later"
|
||||
|
||||
build() {
|
||||
mkdir -p "$builddir"
|
||||
|
||||
# Replace <<INITRAMFS_PKG_VERSION>> with the actual version
|
||||
sed "s|<<INITRAMFS_PKG_VERSION>>|$pkgver-r$pkgrel|" \
|
||||
"$srcdir/init.sh" > "$builddir/init.sh"
|
||||
}
|
||||
|
||||
package() {
|
||||
install -Dm644 "$srcdir/unudhcpd.conf" \
|
||||
"$pkgdir/etc/unudhcpd.conf"
|
||||
|
|
@ -46,7 +54,7 @@ package() {
|
|||
install -Dm644 "$srcdir/init_functions.sh" \
|
||||
"$pkgdir/usr/share/initramfs/init_functions.sh"
|
||||
|
||||
install -Dm755 "$srcdir/init.sh" \
|
||||
install -Dm755 "$builddir/init.sh" \
|
||||
"$pkgdir/usr/share/initramfs/init.sh"
|
||||
|
||||
install -Dm644 "$srcdir"/00-initramfs-base.dirs \
|
||||
|
|
@ -79,8 +87,8 @@ sha512sums="
|
|||
5b364300f31c91fd0591eb0715f67cbf5383f45246a5fb9f34b79f7cb2e3b15768b2130e5f32f816cc169950f988c1beabc879ba31645c58ce131a288dbc071d 00-initramfs-base.dirs
|
||||
ab41b45b0613f25a61114ed8c8b92bc53c60838f6e2e0ba18c76e5369b2984e6023a0661887692673aca3f647f268c468a468f6b1ac424cfee609017a89481dd 00-initramfs-base.files
|
||||
8a4adad3785af474b36a09a05f6a3b2c4b4f43aac331a53b903abfa51ea12be1e3d1d807b7a6e66a1346815f3b0044daf8cd62e21e2dc75d2db13ee265a72985 00-initramfs-extra-base.files
|
||||
701b6252a536dfc7bddef0f5c3ccb16ad680f70efbad872b74d61ef24c98edcf623b19b5e1a2973bc3fa66ca009a515d7ba3563f0d417306a0a2b5680bee7efd init.sh
|
||||
74cfd8e30811bcf3bb075bcc4db0b59c4f63ba83552cfe371132dacd4c41d73c04c65d309609578369243e9b969feba56b4bdd478eb5e7ebff7099e84f7c0f01 init_functions.sh
|
||||
4bfcb41b2d5ccd577a6203a9479e5bdf056064d20f02f9120f1a074c7c280c0231cf380ed6be7c80733ed4ae101385c14f123a8a41780bbb7d0851387cd9c613 init.sh
|
||||
58ee2b36b858c9c70fabbcb9a12a754afc97e6dfe09ddb7a2ec02c6fb593cc40677ae7049ba5ffda7797cba20e4d35f0d320c5fb4d3072cdbdf172cce6665bb0 init_functions.sh
|
||||
ba3275a9af788c7c782322a22a0f144d5e50e3498ea6886486a29331f23ae89cd32d500a3635cfa7cab369afba92edc18aeca64ccbf0cd589061cce23d15b46c unudhcpd.conf
|
||||
675e7d5bee39b2df7d322117f8dcaccc274d61beaf4d50ead19bbf2109446d64b1c0aa0c5b4f9846eb6c1c403418f28f6364eff4537ba41120fbfcbc484b7da7 mdev.conf
|
||||
"
|
||||
|
|
|
|||
|
|
@ -14,6 +14,9 @@ IN_CI="false"
|
|||
# provide a default for os-release's VERSION in case the file doesn't exist
|
||||
VERSION="${VERSION:-unknown}"
|
||||
|
||||
# This is set during packaging and is used when triaging bug reports
|
||||
INITRAMFS_PKG_VERSION="<<INITRAMFS_PKG_VERSION>>"
|
||||
|
||||
export PATH=/usr/bin:/bin:/usr/sbin:/sbin
|
||||
/bin/busybox --install -s
|
||||
/bin/busybox-extras --install -s
|
||||
|
|
@ -40,7 +43,7 @@ run_hooks /hooks
|
|||
if [ "$IN_CI" = "true" ]; then
|
||||
echo "PMOS: CI tests done, disabling console and looping forever"
|
||||
dmesg -n 1
|
||||
loop_forever
|
||||
fail_halt_boot
|
||||
fi
|
||||
|
||||
# Always run dhcp daemon/usb networking for now (later this should only
|
||||
|
|
@ -54,6 +57,19 @@ extract_initramfs_extra /boot/initramfs-extra
|
|||
setup_udev
|
||||
run_hooks /hooks-extra
|
||||
|
||||
# For testing the mass storage gadget log export function. We use a flag
|
||||
# file on /boot so that we can test it on all devices as modifying the
|
||||
# kernel cmdline is not always possible.
|
||||
if [ -e /boot/.pmos_export_logs ]; then
|
||||
echo "PMOS: Exporting logs via mass storage gadget"
|
||||
show_splash "Exporting boot logs..."
|
||||
# Delete the flag so we don't soft-brick the device by always booting
|
||||
# to the log export mode.
|
||||
mount -o remount,rw /boot
|
||||
rm -f /boot/.pmos_export_logs
|
||||
fail_halt_boot
|
||||
fi
|
||||
|
||||
wait_root_partition
|
||||
delete_old_install_partition
|
||||
resize_root_partition
|
||||
|
|
@ -77,4 +93,4 @@ exec switch_root /sysroot "$init"
|
|||
echo "ERROR: switch_root failed!"
|
||||
echo "Looping forever. Install and use the debug-shell hook to debug this."
|
||||
echo "For more information, see <https://postmarketos.org/debug-shell>"
|
||||
loop_forever
|
||||
fail_halt_boot
|
||||
|
|
|
|||
|
|
@ -304,7 +304,7 @@ mount_boot_partition() {
|
|||
if [ -z "$partition" ]; then
|
||||
echo "ERROR: boot partition not found!"
|
||||
show_splash "ERROR: Boot partition not found\\nhttps://postmarketos.org/troubleshooting"
|
||||
loop_forever
|
||||
fail_halt_boot
|
||||
fi
|
||||
|
||||
if [ "$2" = "rw" ]; then
|
||||
|
|
@ -341,7 +341,7 @@ extract_initramfs_extra() {
|
|||
if [ ! -e "$initramfs_extra" ]; then
|
||||
echo "ERROR: initramfs-extra not found!"
|
||||
show_splash "ERROR: initramfs-extra not found\\nhttps://postmarketos.org/troubleshooting"
|
||||
loop_forever
|
||||
fail_halt_boot
|
||||
fi
|
||||
echo "Extract $initramfs_extra"
|
||||
# uncompressed:
|
||||
|
|
@ -513,7 +513,7 @@ mount_root_partition() {
|
|||
if ! { [ "$type" = "ext4" ] || [ "$type" = "f2fs" ] || [ "$type" = "btrfs" ]; } then
|
||||
echo "ERROR: Detected unsupported '$type' filesystem ($partition)."
|
||||
show_splash "ERROR: unsupported '$type' filesystem ($partition)\\nhttps://postmarketos.org/troubleshooting"
|
||||
loop_forever
|
||||
fail_halt_boot
|
||||
fi
|
||||
|
||||
if ! modprobe "$type"; then
|
||||
|
|
@ -522,13 +522,13 @@ mount_root_partition() {
|
|||
if ! mount -t "$type" -o ro"$rootfsopts" "$partition" /sysroot; then
|
||||
echo "ERROR: unable to mount root partition!"
|
||||
show_splash "ERROR: unable to mount root partition\\nhttps://postmarketos.org/troubleshooting"
|
||||
loop_forever
|
||||
fail_halt_boot
|
||||
fi
|
||||
|
||||
if ! [ -e /sysroot/usr ]; then
|
||||
echo "ERROR: root partition appeared to mount but does not contain a root filesystem!"
|
||||
show_splash "ERROR: root partition does not contain a root filesystem\\nhttps://postmarketos.org/troubleshooting"
|
||||
loop_forever
|
||||
fail_halt_boot
|
||||
fi
|
||||
}
|
||||
|
||||
|
|
@ -584,9 +584,11 @@ setup_usb_configfs_udc() {
|
|||
echo "$_udc_dev" > /config/usb_gadget/g1/UDC || echo " Couldn't write new UDC"
|
||||
}
|
||||
|
||||
# $1: if set, skip writing to the UDC
|
||||
setup_usb_network_configfs() {
|
||||
# See: https://www.kernel.org/doc/Documentation/usb/gadget_configfs.txt
|
||||
CONFIGFS=/config/usb_gadget
|
||||
local skip_udc="$1"
|
||||
|
||||
if ! [ -e "$CONFIGFS" ]; then
|
||||
echo " /config/usb_gadget does not exist, skipping configfs usb gadget"
|
||||
|
|
@ -638,7 +640,11 @@ setup_usb_network_configfs() {
|
|||
ln -s $CONFIGFS/g1/functions/"$usb_network_function" $CONFIGFS/g1/configs/c.1 \
|
||||
|| echo " Couldn't symlink $usb_network_function"
|
||||
|
||||
# If an argument was supplied then skip writing to the UDC (only used for mass storage
|
||||
# log recovery)
|
||||
if [ -z "$skip_udc" ]; then
|
||||
setup_usb_configfs_udc
|
||||
fi
|
||||
}
|
||||
|
||||
setup_usb_network() {
|
||||
|
|
@ -771,7 +777,98 @@ setup_bootchart2() {
|
|||
fi
|
||||
}
|
||||
|
||||
loop_forever() {
|
||||
mkhash() {
|
||||
sha256sum "$1" | cut -d " " -f 1
|
||||
}
|
||||
|
||||
# Create a small disk image and copy logs to it so they can be exposed via mass storage
|
||||
create_logs_disk() {
|
||||
local loop_dev="$1"
|
||||
local upload_file=""
|
||||
echo "Creating logs disk"
|
||||
|
||||
fallocate -l 10M /tmp/logs.img
|
||||
# The log device used is assumed to be $loop_dev
|
||||
losetup -f /tmp/logs.img
|
||||
mkfs.vfat -n "PMOS_LOGS" "$loop_dev"
|
||||
mkdir -p /tmp/logs
|
||||
mount "$loop_dev" /tmp/logs
|
||||
|
||||
# Copy logs
|
||||
cp /pmOS_init.log /tmp/logs/pmOS_init.txt
|
||||
dmesg > /tmp/logs/dmesg.txt
|
||||
blkid > /tmp/logs/blkid.txt
|
||||
cat /proc/cmdline > /tmp/logs/cmdline.txt
|
||||
cat /proc/partitions > /tmp/logs/partitions.txt
|
||||
# Include FDT if it exists
|
||||
[ -e /sys/firmware/fdt ] && cp /sys/firmware/fdt /tmp/logs/fdt.dtb
|
||||
|
||||
# Additional info about the initramfs
|
||||
{
|
||||
echo "initramfs-version: $INITRAMFS_PKG_VERSION"
|
||||
# Take hashes of the initramfs files so we can be sure they weren't modified inadvertantly
|
||||
echo "init-hash: $(mkhash /init)"
|
||||
echo "init-functions-hash: $(mkhash /init_functions.sh)"
|
||||
} >> /tmp/logs/_info
|
||||
|
||||
# Create a tar file with all the logs. We don't include the date because on many devices
|
||||
# (especially Qualcomm) the RTC is likely wrong.
|
||||
upload_file="${deviceinfo_codename}-${VERSION}-$(uname -r).tar.gz"
|
||||
# Done in a subshell to not change the working directory of init
|
||||
(cd /tmp/logs || ( echo "Couldn't cd to /tmp/logs"; return ); tar -cv ./* | gzip -6 -c > "/tmp/$upload_file")
|
||||
mv "/tmp/$upload_file" /tmp/logs/
|
||||
|
||||
# Create a README with instructions on how to report an issue
|
||||
cat > /tmp/logs/README.txt <<-EOF
|
||||
Something went wrong and your device did not boot properly. If this was unexpected
|
||||
then please open a new issue by visiting
|
||||
|
||||
https://gitlab.com/postmarketOS/pmaports/-/issues/new
|
||||
|
||||
and attach the following file by dragging it onto the page:
|
||||
|
||||
* $upload_file
|
||||
|
||||
You are running postmarketOS $VERSION on kernel $(uname -r).
|
||||
EOF
|
||||
|
||||
# Unmount
|
||||
umount /tmp/logs
|
||||
}
|
||||
|
||||
# Make logs available via mass storage gadget
|
||||
export_logs() {
|
||||
local loop_dev=""
|
||||
usb_mass_storage_function="mass_storage.0"
|
||||
active_udc="$(cat /config/usb_gadget/g1/UDC)"
|
||||
|
||||
loop_dev="$(losetup -f)"
|
||||
|
||||
create_logs_disk "$loop_dev"
|
||||
|
||||
echo "Making logs available via mass storage"
|
||||
|
||||
# Set up network gadget if not already done
|
||||
if [ -z "$active_udc" ]; then
|
||||
setup_usb_network_configfs "skip_udc"
|
||||
else
|
||||
# Unset UDC
|
||||
echo "" > /config/usb_gadget/g1/UDC
|
||||
fi
|
||||
|
||||
mkdir "$CONFIGFS"/g1/functions/"$usb_mass_storage_function" || return
|
||||
|
||||
echo "$loop_dev" > "$CONFIGFS"/g1/functions/"$usb_mass_storage_function"/lun.0/file
|
||||
|
||||
ln -s "$CONFIGFS"/g1/functions/"$usb_mass_storage_function" \
|
||||
"$CONFIGFS"/g1/configs/c.1 || return
|
||||
|
||||
setup_usb_configfs_udc
|
||||
}
|
||||
|
||||
fail_halt_boot() {
|
||||
export_logs
|
||||
echo "Looping forever"
|
||||
while true; do
|
||||
sleep 1
|
||||
done
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ echo "Create 'pmos_continue_boot' script"
|
|||
#Disable any active usb mass storage
|
||||
echo "if [ -d /config/usb_gadget/g1/functions/mass_storage.0 ]; then setup_usb_storage; fi"
|
||||
echo "pkill -9 -f pmos_shell"
|
||||
echo "pkill -f pmos_loop_forever"
|
||||
echo "pkill -f pmos_fail_halt_boot"
|
||||
echo "pkill -f telnetd.*:${TELNET_PORT}"
|
||||
} >/usr/bin/pmos_continue_boot
|
||||
chmod +x /usr/bin/pmos_continue_boot
|
||||
|
|
@ -27,13 +27,13 @@ echo "Create 'pmos_shell' script"
|
|||
} >/usr/bin/pmos_shell
|
||||
chmod +x /usr/bin/pmos_shell
|
||||
|
||||
echo "Create 'pmos_loop_forever' script"
|
||||
echo "Create 'pmos_fail_halt_boot' script"
|
||||
{
|
||||
echo "#!/bin/sh"
|
||||
echo '. /init_functions.sh'
|
||||
echo "loop_forever"
|
||||
} >/usr/bin/pmos_loop_forever
|
||||
chmod +x /usr/bin/pmos_loop_forever
|
||||
echo "fail_halt_boot"
|
||||
} >/usr/bin/pmos_fail_halt_boot
|
||||
chmod +x /usr/bin/pmos_fail_halt_boot
|
||||
|
||||
echo "Start the telnet daemon"
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
pkgname=postmarketos-mkinitfs-hook-debug-shell
|
||||
pkgver=0.5.0
|
||||
pkgver=0.6.0
|
||||
pkgrel=0
|
||||
pkgdesc="Root shell in the initramfs (security hole, for debugging only)"
|
||||
url="https://postmarketos.org"
|
||||
|
|
@ -19,7 +19,7 @@ package() {
|
|||
}
|
||||
|
||||
sha512sums="
|
||||
0750a33f3b68fd6f751bdcf3c59656ec8c8af60e13c25b13394937134d4cbeaacd237b5d92e2a978319f705781f39712c71673367cec52b584f666af50b6f952 20-debug-shell.sh
|
||||
add95b3fa64b804fd386298406fbc55489e3e122d70f40e45627a450d299e148a346a4a147c2b28e0110ecef14e9e0b62ac75a0bd2d39295c2407ebb5e93708c 20-debug-shell.sh
|
||||
a739bc47d905d189edb26d9ebfd062023720fefdaab27207471c16d53a9c12ea8b81092a1047d8f2300e42ba500bdf6c5a3343aca55aab5bf8e84d68eb5680ab 20-debug-shell.files
|
||||
75d485c2e9f352cfd717b7a92753a9dfc4a72526a44bcbb784eacb4ef9011072b3ffa1c42a317c0940598cc076fb6c61676c440e5b188378b19ca08d882c1338 setup_usb_storage.sh
|
||||
"
|
||||
|
|
|
|||
|
|
@ -53,4 +53,4 @@ vibrate_loop() {
|
|||
blink_leds $(find_leds) &
|
||||
vibrate_loop $(find_vibrator) &
|
||||
|
||||
loop_forever
|
||||
fail_halt_boot
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
pkgname=postmarketos-mkinitfs-hook-maximum-attention
|
||||
pkgver=0.1.0
|
||||
pkgrel=1
|
||||
pkgver=0.2.0
|
||||
pkgrel=0
|
||||
pkgdesc="Script to activate all user-visible outputs from the initramfs (to confirm working kernel, for debugging only)"
|
||||
url="https://postmarketos.org"
|
||||
depends="postmarketos-mkinitfs"
|
||||
|
|
@ -12,4 +12,6 @@ package() {
|
|||
install -Dm644 "$srcdir"/00-maximum-attention.sh \
|
||||
-t "$pkgdir"/usr/share/mkinitfs/hooks/
|
||||
}
|
||||
sha512sums="c29cc41649c284c0ec8201715cd9452f838801a22ea6c2a312ea87a58c61c1caa05a3b1ff33ce4a4d680a95e79fb10108308b72173f1fa97bac5cefde7ab30ee 00-maximum-attention.sh"
|
||||
sha512sums="
|
||||
ca3ef48e76e21fdb67fdf7477fb4739e329ce6ea04a52ac14e9365adb92058b21e08fcbc4020519c61ba20cd3c91d254118ece3d1818cb46f86e86f5ffa1b90a 00-maximum-attention.sh
|
||||
"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Maintainer: Mark <clashclanacc2602@gmail.com>
|
||||
pkgname=postmarketos-mkinitfs-hook-netboot
|
||||
pkgver=4.0.3
|
||||
pkgver=4.1.3
|
||||
pkgrel=0
|
||||
pkgdesc="Netboot postmarketOS using nbd feature"
|
||||
url="https://postmarketos.org"
|
||||
|
|
@ -21,6 +21,6 @@ package() {
|
|||
}
|
||||
|
||||
sha512sums="
|
||||
039b6ae08ffb2e24573a5659fef8ff14f70219e06fe6dd6c5ad459f3b54883e17166060813762cdc71b7c774f450956158871fd64c70330d0a661a8cc94666e7 netboot.sh
|
||||
c681d0c39eecd9fe5d284e07f9c6f79e41085628abe1c645b8ea3cc4e621fd603fca9db4a555fa689a3268925cb6d23c12baa98986baed452375acd1e3fe26b1 netboot.sh
|
||||
3c9c812a6cd25640e6cf4d722cff6c6356d5666ec6009b4c65f89e62211cc406d2cd09e0fdda6b1c1906ee32c3854988ab701cee8811208e3ef666f09e638568 netboot.modules
|
||||
"
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ modprobe nbd
|
|||
if [ ! -b /dev/nbd0 ]; then
|
||||
echo "Failed to get /dev/nbd0, stopping."
|
||||
show_splash "ERROR: Failed to initialise netboot\\nhttps://postmarketos.org/netboot"
|
||||
pmos_loop_forever
|
||||
fail_halt_boot
|
||||
fi
|
||||
|
||||
client_ip="${unudhcpd_client_ip:-172.16.42.2}"
|
||||
|
|
|
|||
Loading…
Reference in a new issue