From 12460fcea6cc470dec1d2bbf00eadb5adc9ed228 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 8 Aug 2018 12:03:30 -0400 Subject: [PATCH] make --batch honor matching options When --batch is used with matching options like --in, --metadata, etc, only operate on the provided files when they match those options. Otherwise, a blank line is output in the batch protocol. Affected commands: find, add, whereis, drop, copy, move, get In the case of find, the documentation for --batch already said it honored the matching options. The docs for the rest didn't, but it makes sense to have them honor them. While this is a behavior change, why specify the matching options with --batch if you didn't want them to apply? Note that the batch output for all of the affected commands could already output a blank line in other cases, so batch users should already be prepared to deal with it. git-annex metadata didn't seem worth making support the matching options, since all it does is output metadata or set metadata, the use cases for using it in combination with the martching options seem small. Made it refuse to run when they're combined, leaving open the possibility for later support if a use case develops. This commit was sponsored by Brett Eisenberg on Patreon. --- CHANGELOG | 11 +++++++++++ CmdLine/Batch.hs | 19 +++++++++++++++++-- Command/Add.hs | 2 +- Command/Copy.hs | 2 +- Command/Drop.hs | 2 +- Command/Find.hs | 2 +- Command/Get.hs | 2 +- Command/MetaData.hs | 8 ++++++-- Command/Move.hs | 2 +- Command/Whereis.hs | 2 +- ...batch_does_not_apply_matching_options.mdwn | 3 +++ doc/git-annex-add.mdwn | 5 +++-- doc/git-annex-copy.mdwn | 5 +++-- doc/git-annex-drop.mdwn | 5 +++++ doc/git-annex-get.mdwn | 3 ++- doc/git-annex-metadata.mdwn | 3 +++ doc/git-annex-move.mdwn | 3 ++- doc/git-annex-whereis.mdwn | 3 ++- 18 files changed, 64 insertions(+), 18 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 2b011e0112..fcc69d1ee8 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,14 @@ +git-annex (6.20180808) UNRELEASED; urgency=medium + + * When --batch is used with matching options like --in, --metadata, + etc, only operate on the provided files when they match those options. + Otherwise, a blank line is output in the batch protocol. + Affected commands: find, add, whereis, drop, copy, move, get + * Make metadata --batch combined with matching options refuse to run, + since it does not seem worth supporting that combination. + + -- Joey Hess Wed, 08 Aug 2018 11:24:08 -0400 + git-annex (6.20180807) upstream; urgency=medium * S3: Support credential-less download from remotes configured diff --git a/CmdLine/Batch.hs b/CmdLine/Batch.hs index 82038314c0..cea108f126 100644 --- a/CmdLine/Batch.hs +++ b/CmdLine/Batch.hs @@ -12,6 +12,8 @@ import Types.Command import CmdLine.Action import CmdLine.GitAnnex.Options import Options.Applicative +import Limit +import Types.FileMatcher data BatchMode = Batch | NoBatch @@ -72,5 +74,18 @@ batchCommandAction a = maybe (batchBadInput Batch) (const noop) -- Reads lines of batch input and passes the filepaths to a CommandStart -- to handle them. -batchFiles :: (FilePath -> CommandStart) -> Annex () -batchFiles a = batchInput Right $ batchCommandAction . a +-- +-- File matching options are not checked. +allBatchFiles :: (FilePath -> CommandStart) -> Annex () +allBatchFiles a = batchInput Right $ batchCommandAction . a + +-- Like allBatchFiles, but checks the file matching options +-- and skips non-matching files. +batchFilesMatching :: (FilePath -> CommandStart) -> Annex () +batchFilesMatching a = do + matcher <- getMatcher + allBatchFiles $ \f -> + ifM (matcher $ MatchingFile $ FileInfo f f) + ( a f + , return Nothing + ) diff --git a/Command/Add.hs b/Command/Add.hs index 10148ad50a..de602a9a7c 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -62,7 +62,7 @@ seek o = allowConcurrentOutput $ do Batch | updateOnly o -> giveup "--update --batch is not supported" - | otherwise -> batchFiles gofile + | otherwise -> batchFilesMatching gofile NoBatch -> do l <- workTreeItems (addThese o) let go a = a gofile l diff --git a/Command/Copy.hs b/Command/Copy.hs index daf2e66bcb..d3248f42c6 100644 --- a/Command/Copy.hs +++ b/Command/Copy.hs @@ -47,7 +47,7 @@ seek :: CopyOptions -> CommandSeek seek o = allowConcurrentOutput $ do let go = whenAnnexed $ start o case batchOption o of - Batch -> batchInput Right (batchCommandAction . go) + Batch -> batchFilesMatching go NoBatch -> withKeyOptions (keyOptions o) (autoMode o) (Command.Move.startKey (fromToOptions o) Command.Move.RemoveNever) diff --git a/Command/Drop.hs b/Command/Drop.hs index baeae66eeb..4d7f13f687 100644 --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -54,7 +54,7 @@ parseDropFromOption = parseRemoteOption <$> strOption seek :: DropOptions -> CommandSeek seek o = allowConcurrentOutput $ case batchOption o of - Batch -> batchInput Right (batchCommandAction . go) + Batch -> batchFilesMatching go NoBatch -> withKeyOptions (keyOptions o) (autoMode o) (startKeys o) (withFilesInGit go) diff --git a/Command/Find.hs b/Command/Find.hs index 10eff3527a..9d7c040d27 100644 --- a/Command/Find.hs +++ b/Command/Find.hs @@ -51,7 +51,7 @@ parseFormatOption = seek :: FindOptions -> CommandSeek seek o = case batchOption o of NoBatch -> withFilesInGit go =<< workTreeItems (findThese o) - Batch -> batchFiles go + Batch -> batchFilesMatching go where go = whenAnnexed $ start o diff --git a/Command/Get.hs b/Command/Get.hs index eac8e88a44..fde65c5018 100644 --- a/Command/Get.hs +++ b/Command/Get.hs @@ -42,7 +42,7 @@ seek o = allowConcurrentOutput $ do from <- maybe (pure Nothing) (Just <$$> getParsed) (getFrom o) let go = whenAnnexed $ start o from case batchOption o of - Batch -> batchInput Right (batchCommandAction . go) + Batch -> batchFilesMatching go NoBatch -> withKeyOptions (keyOptions o) (autoMode o) (startKeys from) (withFilesInGit go) diff --git a/Command/MetaData.hs b/Command/MetaData.hs index 282b7fda05..23f16a53a0 100644 --- a/Command/MetaData.hs +++ b/Command/MetaData.hs @@ -15,6 +15,7 @@ import Annex.WorkTree import Messages.JSON (JSONActionItem(..)) import Types.Messages import Utility.Aeson +import Limit import qualified Data.Set as S import qualified Data.Map as M @@ -83,8 +84,11 @@ seek o = case batchOption o of (seeker $ whenAnnexed $ start c o) =<< workTreeItems (forFiles o) Batch -> withMessageState $ \s -> case outputType s of - JSONOutput _ -> batchInput parseJSONInput $ - commandAction . startBatch + JSONOutput _ -> ifM limited + ( giveup "combining --batch with file matching options is not currently supported" + , batchInput parseJSONInput $ + commandAction . startBatch + ) _ -> giveup "--batch is currently only supported in --json mode" start :: VectorClock -> MetaDataOptions -> FilePath -> Key -> CommandStart diff --git a/Command/Move.hs b/Command/Move.hs index b50c877bcc..f5de2c9636 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -57,7 +57,7 @@ seek :: MoveOptions -> CommandSeek seek o = allowConcurrentOutput $ do let go = whenAnnexed $ start (fromToOptions o) (removeWhen o) case batchOption o of - Batch -> batchInput Right (batchCommandAction . go) + Batch -> batchFilesMatching go NoBatch -> withKeyOptions (keyOptions o) False (startKey (fromToOptions o) (removeWhen o)) (withFilesInGit go) diff --git a/Command/Whereis.hs b/Command/Whereis.hs index b14e231c17..988c4aaf5a 100644 --- a/Command/Whereis.hs +++ b/Command/Whereis.hs @@ -40,7 +40,7 @@ seek o = do m <- remoteMap id let go = whenAnnexed $ start m case batchOption o of - Batch -> batchFiles go + Batch -> batchFilesMatching go NoBatch -> withKeyOptions (keyOptions o) False (startKeys m) diff --git a/doc/bugs/find_with_batch_does_not_apply_matching_options.mdwn b/doc/bugs/find_with_batch_does_not_apply_matching_options.mdwn index 767f1db481..813ae9caf1 100644 --- a/doc/bugs/find_with_batch_does_not_apply_matching_options.mdwn +++ b/doc/bugs/find_with_batch_does_not_apply_matching_options.mdwn @@ -8,3 +8,6 @@ Using `git annex find --batch` with matching options seems to not apply them. ### What version of git-annex are you using? On what operating system? I'd rather not say ~~because it is ancient~~. Joey says this is reproducible on recent git-annex versions though. + +> Not only find, but a bunch of commands supporting --batch had this +> oversight. Fixed them all. [[done]] --[[Joey]] diff --git a/doc/git-annex-add.mdwn b/doc/git-annex-add.mdwn index ff7bc40045..f24ec761d5 100644 --- a/doc/git-annex-add.mdwn +++ b/doc/git-annex-add.mdwn @@ -77,8 +77,9 @@ annexed content, and other symlinks. the file is added, and repeat. Note that if a file is skipped (due to not existing, being gitignored, - already being in git etc), an empty line will be output instead of the - normal output produced when adding a file. + already being in git, or doesn't meet the matching options), + an empty line will be output instead of the normal output produced + when adding a file. # SEE ALSO diff --git a/doc/git-annex-copy.mdwn b/doc/git-annex-copy.mdwn index fedeaa067e..9a5b48be98 100644 --- a/doc/git-annex-copy.mdwn +++ b/doc/git-annex-copy.mdwn @@ -80,8 +80,9 @@ Copies the content of files from or to another remote. are read from stdin. As each specified file is processed, the usual progress output is - displayed. If a file's content does not need to be copied or it - is not an annexed file, a blank line is output in response instead. + displayed. If a file's content does not need to be copied, or it does not + match specified matching options, or it is not an annexed file, + a blank line is output in response instead. Since the usual output while copying a file is verbose and not machine-parseable, you may want to use --json in combination with diff --git a/doc/git-annex-drop.mdwn b/doc/git-annex-drop.mdwn index 6975e4a08d..7074ca5cc9 100644 --- a/doc/git-annex-drop.mdwn +++ b/doc/git-annex-drop.mdwn @@ -82,6 +82,11 @@ safe to do so. Enables batch mode, in which lines containing names of files to drop are read from stdin. + As each specified file is processed, the usual output is + displayed. If a file's content is not present, or it does not + match specified matching options, or it is not an annexed file, + a blank line is output in response instead. + * `--json` Enable JSON output. This is intended to be parsed by programs that use diff --git a/doc/git-annex-get.mdwn b/doc/git-annex-get.mdwn index 89c5388fb6..1287fa13d7 100644 --- a/doc/git-annex-get.mdwn +++ b/doc/git-annex-get.mdwn @@ -90,7 +90,8 @@ or transferring them from some kind of key-value store. are read from stdin. As each specified file is processed, the usual progress output is - displayed. If the specified file's content is already present, or + displayed. If the specified file's content is already present, + or it does not match specified matching options, or it is not an annexed file, a blank line is output in response instead. Since the usual output while getting a file is verbose and not diff --git a/doc/git-annex-metadata.mdwn b/doc/git-annex-metadata.mdwn index 5e86077cc1..a672b104ce 100644 --- a/doc/git-annex-metadata.mdwn +++ b/doc/git-annex-metadata.mdwn @@ -152,6 +152,9 @@ automatically. {"file":"foo","fields":{"author":[]}} + Note that file matching options do not affect the files that are + processed when in batch mode. + # EXAMPLES To set some tags on a file and also its author: diff --git a/doc/git-annex-move.mdwn b/doc/git-annex-move.mdwn index 661edba1d9..3af858f8df 100644 --- a/doc/git-annex-move.mdwn +++ b/doc/git-annex-move.mdwn @@ -80,7 +80,8 @@ Moves the content of files from or to another remote. are read from stdin. As each specified file is processed, the usual progress output is - displayed. If a file's content does not need to be moved or it + displayed. If a file's content does not need to be moved, + or it does not match specified matching options, or it is not an annexed file, a blank line is output in response instead. Since the usual output while moving a file is verbose and not diff --git a/doc/git-annex-whereis.mdwn b/doc/git-annex-whereis.mdwn index 9822c57039..5c3a774ef3 100644 --- a/doc/git-annex-whereis.mdwn +++ b/doc/git-annex-whereis.mdwn @@ -48,7 +48,8 @@ For example: Enables batch mode, in which a file is read in a line from stdin, its information displayed, and repeat. - Note that if the file is not an annexed file, an empty line will be + Note that if the file is not an annexed file, or does not match + specified file matching options, an empty line will be output instead. * `--json`