From 7919349ceefff029bcecaeb0efba9cbb5c9deda9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 9 May 2023 16:43:16 -0400 Subject: [PATCH] importfeed: Support --json and --json-error-messages and --json-progress Sponsored-By: the NIH-funded NICEMAN (ReproNim TR&D3) project --- CHANGELOG | 7 +++--- Command/ImportFeed.hs | 25 +++++++++++-------- doc/git-annex-importfeed.mdwn | 14 +++++++++++ ...annex__and_ideally_any_other_command_.mdwn | 5 +--- 4 files changed, 34 insertions(+), 17 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 3ff51db49d..fb19b60898 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -38,15 +38,16 @@ git-annex (10.20230408) UNRELEASED; urgency=medium * initremote: Avoid creating a remote that is not encrypted when gpg is broken. * Support --json and --json-error-messages in more commands - (addunused, dead, describe, dropunused, expire, fix, init, log, migrate, - reinit, reinject, rekey, renameremote, rmurl, semitrust, setpresentkey, - trust, unannex, undo, untrust, unused) + (addunused, dead, describe, dropunused, expire, fix, importfeed, init, + log, migrate, reinit, reinject, rekey, renameremote, rmurl, semitrust, + setpresentkey, trust, unannex, undo, untrust, unused) * log: When --raw-date is used, display only seconds from the epoch, as documented, omitting a trailing "s" that was included in the output before. * addunused: Displays the names of the files that it adds. * reinject: Fix support for operating on multiple pairs of files and keys. * importfeed: Support -J + * importfeed: Support --json-progress -- Joey Hess Sat, 08 Apr 2023 13:57:18 -0400 diff --git a/Command/ImportFeed.hs b/Command/ImportFeed.hs index 55a8564703..2901e7628f 100644 --- a/Command/ImportFeed.hs +++ b/Command/ImportFeed.hs @@ -54,9 +54,11 @@ import Logs import qualified Utility.RawFilePath as R cmd :: Command -cmd = notBareRepo $ withAnnexOptions [jobsOption, backendOption] $ +cmd = notBareRepo $ withAnnexOptions os $ command "importfeed" SectionCommon "import files from podcast feeds" (paramRepeating paramUrl) (seek <$$> optParser) + where + os = [jobsOption, jsonOptions, jsonProgressOption, backendOption] data ImportFeedOptions = ImportFeedOptions { feedUrls :: CmdParams @@ -139,7 +141,7 @@ getFeed -> TMVar (M.Map URLString (Maybe (Maybe [ToDownload]))) -> CommandStart getFeed url st = - starting "importfeed" (ActionItemOther (Just (UnquotedString url))) (SeekInput []) $ + starting "importfeed" (ActionItemOther (Just (UnquotedString url))) (SeekInput [url]) $ get `onException` recordfail where record v = liftIO $ atomically $ do @@ -210,9 +212,12 @@ getCache :: Maybe String -> Annex Cache getCache opttemplate = ifM (Annex.getRead Annex.force) ( ret S.empty S.empty , do - showStartMessage (StartMessage "importfeed" (ActionItemOther (Just "gathering known urls")) (SeekInput [])) + j <- jsonOutputEnabled + unless j $ + showStartMessage (StartMessage "importfeed" (ActionItemOther (Just "gathering known urls")) (SeekInput [])) (us, is) <- knownItems - showEndOk + unless j + showEndOk ret (S.fromList us) (S.fromList is) ) where @@ -295,7 +300,7 @@ startDownload addunlockedmatcher opts cache cv todownload = case location todown recordsuccess = liftIO $ atomically $ putTMVar cv True startdownloadenclosure :: URLString -> CommandStart - startdownloadenclosure url = checkknown url $ startUrlDownload cv url $ + startdownloadenclosure url = checkknown url $ startUrlDownload cv todownload url $ downloadEnclosure addunlockedmatcher opts cache cv todownload url knownitemid = case getItemId (item todownload) of @@ -306,7 +311,7 @@ startDownload addunlockedmatcher opts cache cv todownload = case location todown downloadmedia linkurl mediaurl mediakey | rawOption (downloadOptions opts) = startdownloadlink | otherwise = ifM (youtubeDlSupported linkurl) - ( startUrlDownload cv linkurl $ + ( startUrlDownload cv todownload linkurl $ withTmpWorkDir mediakey $ \workdir -> do dl <- youtubeDl linkurl (fromRawFilePath workdir) nullMeterUpdate case dl of @@ -336,7 +341,7 @@ startDownload addunlockedmatcher opts cache cv todownload = case location todown addmediafast linkurl mediaurl mediakey = ifM (pure (not (rawOption (downloadOptions opts))) <&&> youtubeDlSupported linkurl) - ( startUrlDownload cv linkurl $ do + ( startUrlDownload cv todownload linkurl $ do runDownload todownload linkurl ".m" cache cv $ \f -> checkCanAdd (downloadOptions opts) f $ \canadd -> do addWorkTree canadd addunlockedmatcher webUUID mediaurl f mediakey Nothing @@ -453,10 +458,10 @@ runDownload todownload url extension cache cv getter = do , tryanother ) -startUrlDownload :: TMVar Bool -> URLString -> CommandPerform -> CommandStart -startUrlDownload cv url a = starting "addurl" +startUrlDownload :: TMVar Bool -> ToDownload -> URLString -> CommandPerform -> CommandStart +startUrlDownload cv todownload url a = starting "addurl" (ActionItemOther (Just (UnquotedString url))) - (SeekInput []) + (SeekInput [feedurl todownload]) (a `onException` recordfailure) where recordfailure = do diff --git a/doc/git-annex-importfeed.mdwn b/doc/git-annex-importfeed.mdwn index afcc29a220..aa8e45b62d 100644 --- a/doc/git-annex-importfeed.mdwn +++ b/doc/git-annex-importfeed.mdwn @@ -112,6 +112,20 @@ resulting in the new url being downloaded to such a filename. Specifies which key-value backend to use. +* `--json` + + Enable JSON output. This is intended to be parsed by programs that use + git-annex. Each line of output is a JSON object. + +* `--json-progress` + + Include progress objects in JSON output. + +* `--json-error-messages` + + Messages that would normally be output to standard error are included in + the JSON instead. + * Also the [[git-annex-common-options]](1) can be used. # SEE ALSO diff --git a/doc/todo/--json_for_unannex__and_ideally_any_other_command_.mdwn b/doc/todo/--json_for_unannex__and_ideally_any_other_command_.mdwn index 5ae0d171e3..b165cb6ef7 100644 --- a/doc/todo/--json_for_unannex__and_ideally_any_other_command_.mdwn +++ b/doc/todo/--json_for_unannex__and_ideally_any_other_command_.mdwn @@ -36,6 +36,7 @@ These commands have been updated to support --json: * git-annex-reinit * git-annex-reinject * git-annex-renameremote +* git-annex-importfeed Provisional list of commands that don't support --json and maybe should: @@ -55,10 +56,6 @@ These commands could support json, but I punted: * git-annex-version (--raw already exists, and the output is fairly machine parseable already. It would be possible to jsonize the output to make it possibly more machine parseable. But I'm doubtful that would be useful. -* git-annex-importfeed (implemented w/o using usual command actions, - which makes warning messages not get put in any particular json record. - Same problem would also need to be fixed for [[doc/todo/importfeed_parallell]] - btw.) These commands have been reviewed and should not support json: