From 7a0d6d81a02242b5c8f34ed641113ac241166c0d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 20 Feb 2017 15:59:55 -0400 Subject: [PATCH] make curl show http errors to stderr * Run curl with -S, so HTTP errors are displayed, even when it's otherwise silent. * When downloading in --json or --quiet mode, use curl in preference to wget, since curl is able to display only errors to stderr, unlike wget. This does mean that downloadQuiet is only silent on stdout, not necessarily on stderr, which affects a couple other calls of it. For example, downloading the .git/config of a http remote may show an error message now, perhaps with slightly suboptimal formatting due to other output. --- CHANGELOG | 5 +++++ Utility/Url.hs | 22 ++++++++++++------- ...n_case_of_failures_into_returned_json.mdwn | 2 ++ ..._82b851629c695084cbf62e2b636bcc91._comment | 9 ++++---- 4 files changed, 26 insertions(+), 12 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 4e150f340c..3b631092eb 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -27,6 +27,11 @@ git-annex (6.20170215) UNRELEASED; urgency=medium * adjust: Fix behavior when used in a repository that contains submodules. * Run wget with -nv instead of -q, so it will display HTTP errors. + * Run curl with -S, so HTTP errors are displayed, even when + it's otherwise silent. + * When downloading in --json or --quiet mode, use curl in preference + to wget, since curl is able to display only errors to stderr, unlike + wget. -- Joey Hess Tue, 14 Feb 2017 15:54:25 -0400 diff --git a/Utility/Url.hs b/Utility/Url.hs index 166783e09c..1b68dce7fd 100644 --- a/Utility/Url.hs +++ b/Utility/Url.hs @@ -246,17 +246,16 @@ headRequest r = r (requestHeaders r) } -{- Used to download large files, such as the contents of keys. +{- Download a perhaps large file, with auto-resume of incomplete downloads. - - - Uses wget or curl program for its progress bar. (Wget has a better one, - - so is preferred.) Which program to use is determined at run time; it - - would not be appropriate to test at configure time and build support - - for only one in. + - Uses wget or curl program for its progress bar and resuming support. + - Which program to use is determined at run time depending on which is + - in path and which works best in a particular situation. -} download :: URLString -> FilePath -> UrlOptions -> IO Bool download = download' False -{- No output, even on error. -} +{- No output to stdout. -} downloadQuiet :: URLString -> FilePath -> UrlOptions -> IO Bool downloadQuiet = download' True @@ -265,6 +264,12 @@ download' quiet url file uo = do case parseURIRelaxed url of Just u | uriScheme u == "file:" -> curl + -- curl is preferred in quiet mode, because + -- it displays http errors to stderr, while wget + -- does not display them in quiet mode + | quiet -> ifM (inPath "curl") (curl, wget) + -- wget is preferred mostly because it has a better + -- progress bar | otherwise -> ifM (inPath "wget") (wget , curl) _ -> return False where @@ -275,7 +280,8 @@ download' quiet url file uo = do - support, or need that option. - - When the wget version is new enough, pass options for - - a less cluttered download display. + - a less cluttered download display. Using -nv rather than -q + - avoids most clutter while still displaying http errors. -} #ifndef __ANDROID__ wgetparams = concat @@ -296,7 +302,7 @@ download' quiet url file uo = do -- curl does not create destination file -- if the url happens to be empty, so pre-create. writeFile file "" - go "curl" $ headerparams ++ quietopt "-s" ++ + go "curl" $ headerparams ++ quietopt "-sS" ++ [ Param "-f" , Param "-L" , Param "-C", Param "-" diff --git a/doc/todo/more_of_diagnostic_information_in_case_of_failures_into_returned_json.mdwn b/doc/todo/more_of_diagnostic_information_in_case_of_failures_into_returned_json.mdwn index b9283ae546..3e9c30970e 100644 --- a/doc/todo/more_of_diagnostic_information_in_case_of_failures_into_returned_json.mdwn +++ b/doc/todo/more_of_diagnostic_information_in_case_of_failures_into_returned_json.mdwn @@ -18,3 +18,5 @@ ATM I am experiencing sporadic failures of the batched git annex addurl call -- besides me blindly trying to re-run it e.g. 3 times and only then declare total failure, I wondered if json output could provide more information (if any known) about the failure... e.g. if a custom remote crashed/errorred (I guess the case here due to "from datalad") -- what was stderr/exit code for that process if crashed/ERROR msg... if wget -- what was stderr there [[!meta name=yoh]] + +> Switched to curl with -sS in --json mode. [[done]] I suppose. --[[Joey]] diff --git a/doc/todo/more_of_diagnostic_information_in_case_of_failures_into_returned_json/comment_2_82b851629c695084cbf62e2b636bcc91._comment b/doc/todo/more_of_diagnostic_information_in_case_of_failures_into_returned_json/comment_2_82b851629c695084cbf62e2b636bcc91._comment index 40935530e4..0ece7ca770 100644 --- a/doc/todo/more_of_diagnostic_information_in_case_of_failures_into_returned_json/comment_2_82b851629c695084cbf62e2b636bcc91._comment +++ b/doc/todo/more_of_diagnostic_information_in_case_of_failures_into_returned_json/comment_2_82b851629c695084cbf62e2b636bcc91._comment @@ -3,13 +3,14 @@ subject="""comment 2""" date="2017-02-20T19:15:18Z" content=""" +It seems there is no way to get only errors on +stderr with wget; the choice is between no output, and a mixture of errors +and informational messages on stderr. In --json or --quiet mode, only +errors should be output to stderr. + In general, the --json output does include a "note" with any available message about why an operation failed. -Since wget outputs HTTP errors to stdout, there's no way to capture -the actual message for json, and so swiching to wget -nv won't improve -the json, since in json mode wget is run with -q. - It would not be hard to use a HTTP library and propagate the HTTP errors into the json "note", but it might be hard to get resumption of partial downloads to work as well with a HTTP library as it works with wget/curl.