From 5bc70e2da5fbc974940587a1c8214d5a56e4a64e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 5 Aug 2022 13:57:20 -0400 Subject: [PATCH] When bup split fails, display its stderr It seems worth noting here that I emailed bup's author about bup split being noisy on stderr even with -q in approximately 2011. That never got fixed. Its current repo on github only accepts pull requests, not bug reports. Needing to add such complexity to deal with such a longstanding unfixed issue is not fun. Sponsored-by: Kevin Mueller on Patreon --- CHANGELOG | 1 + Remote/Bup.hs | 24 ++++++++++++------- Utility/Process.hs | 1 + ..._1731cfb2662890299ca6daeacdd4bb47._comment | 22 +++++++++++++++++ 4 files changed, 40 insertions(+), 8 deletions(-) create mode 100644 doc/bugs/bup_often_errors_out_when_-J___62___1/comment_1_1731cfb2662890299ca6daeacdd4bb47._comment diff --git a/CHANGELOG b/CHANGELOG index 45991265a9..f0232c45b7 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -14,6 +14,7 @@ git-annex (10.20220725) UNRELEASED; urgency=medium * Work around bug in git 2.37 that causes a segfault when when core.untrackedCache is set, and broke git-annex init. * Improve output when storing to bup. + * When bup split fails, display its stderr. -- Joey Hess Mon, 25 Jul 2022 15:35:45 -0400 diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 35e95e6bbe..4d007243d3 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -1,6 +1,6 @@ {- Using bup as a remote. - - - Copyright 2011-2020 Joey Hess + - Copyright 2011-2022 Joey Hess - - Licensed under the GNU AGPL version 3 or higher. -} @@ -13,6 +13,7 @@ import qualified Data.Map as M import qualified Data.ByteString as S import qualified Data.ByteString.Lazy as L import Data.ByteString.Lazy.UTF8 (fromString) +import Control.Concurrent.Async import Annex.Common import qualified Annex @@ -160,19 +161,26 @@ store r buprepo = byteStorer $ \k b p -> do cmd = (proc "bup" (toCommand params)) { std_in = CreatePipe , std_out = UseHandle nullh - -- bup split is noisy to stderr even with the -q - -- option. - , std_err = UseHandle nullh + , std_err = CreatePipe } feeder = \h -> do meteredWrite p (S.hPut h) b hClose h in withCreateProcess cmd (go feeder cmd) where - go feeder p (Just h) _ _ pid = - forceSuccessProcess p pid - `after` - feeder h + go feeder p (Just inh) _ (Just errh) pid = do + -- bup split is noisy to stderr even with the -q + -- option. But when bup fails, the stderr needs + -- to be displayed. + (feedresult, erroutput) <- tryNonAsync (feeder inh) + `concurrently` hGetContentsStrict errh + waitForProcess pid >>= \case + ExitSuccess -> case feedresult of + Right () -> return () + Left e -> throwM e + ExitFailure n -> giveup $ + showCmd p ++ " exited " ++ show n ++ + " (stderr output: " ++ erroutput ++ ")" go _ _ _ _ _ _ = error "internal" retrieve :: BupRepo -> Retriever diff --git a/Utility/Process.hs b/Utility/Process.hs index 4cf61054b4..e768c13367 100644 --- a/Utility/Process.hs +++ b/Utility/Process.hs @@ -31,6 +31,7 @@ module Utility.Process ( stdoutHandle, stderrHandle, processHandle, + showCmd, devNull, ) where diff --git a/doc/bugs/bup_often_errors_out_when_-J___62___1/comment_1_1731cfb2662890299ca6daeacdd4bb47._comment b/doc/bugs/bup_often_errors_out_when_-J___62___1/comment_1_1731cfb2662890299ca6daeacdd4bb47._comment new file mode 100644 index 0000000000..6659b61e8d --- /dev/null +++ b/doc/bugs/bup_often_errors_out_when_-J___62___1/comment_1_1731cfb2662890299ca6daeacdd4bb47._comment @@ -0,0 +1,22 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2022-08-05T15:57:26Z" + content=""" +This seems most likely to be a concurrency issue in bup itself, or in how +it uses git. + +However, I have not been able to reproduce it so far. I tried making 8 +different files each 100 mb, and was able to copy them all to a bup repository +concurrently with no failures. + +That was with bup 0.32. What is your bup version? + +It would be good to know what error message bup outputs. +Unfortunately, since `bup split -q` is noisy on stderr, git-annex +sinks it to /dev/null and so you don't see it. + +I've changed git-annex to display the stderr when `bup split` fails. +So if you can get an updated build of git-annex, and reproduce this problem, +we'll be able to see if this is a bug in bup, or what. +"""]]