make --batch honor matching options

When --batch is used with matching options like --in, --metadata, etc, only
operate on the provided files when they match those options. Otherwise, a
blank line is output in the batch protocol.

Affected commands: find, add, whereis, drop, copy, move, get

In the case of find, the documentation for --batch already said it honored
the matching options. The docs for the rest didn't, but it makes sense to
have them honor them. While this is a behavior change, why specify the
matching options with --batch if you didn't want them to apply?

Note that the batch output for all of the affected commands could
already output a blank line in other cases, so batch users should
already be prepared to deal with it.

git-annex metadata didn't seem worth making support the matching options,
since all it does is output metadata or set metadata, the use cases for
using it in combination with the martching options seem small. Made it
refuse to run when they're combined, leaving open the possibility for later
support if a use case develops.

This commit was sponsored by Brett Eisenberg on Patreon.

CHANGELOG

@@ -1,3 +1,14 @@
+git-annex (6.20180808) UNRELEASED; urgency=medium
+
+  * When --batch is used with matching options like --in, --metadata,
+    etc, only operate on the provided files when they match those options.
+    Otherwise, a blank line is output in the batch protocol.
+    Affected commands: find, add, whereis, drop, copy, move, get
+  * Make metadata --batch combined with matching options refuse to run,
+    since it does not seem worth supporting that combination.
+
+ -- Joey Hess <id@joeyh.name>  Wed, 08 Aug 2018 11:24:08 -0400
+
 git-annex (6.20180807) upstream; urgency=medium
 
   * S3: Support credential-less download from remotes configured

CmdLine/Batch.hs

@@ -12,6 +12,8 @@ import Types.Command
 import CmdLine.Action
 import CmdLine.GitAnnex.Options
 import Options.Applicative
+import Limit
+import Types.FileMatcher
 
 data BatchMode = Batch | NoBatch
 
@@ -72,5 +74,18 @@ batchCommandAction a = maybe (batchBadInput Batch) (const noop)
 
 -- Reads lines of batch input and passes the filepaths to a CommandStart
 -- to handle them.
-batchFiles :: (FilePath -> CommandStart) -> Annex ()
+--
-batchFiles a = batchInput Right $ batchCommandAction . a
+-- File matching options are not checked.
+allBatchFiles :: (FilePath -> CommandStart) -> Annex ()
+allBatchFiles a = batchInput Right $ batchCommandAction . a
+
+-- Like allBatchFiles, but checks the file matching options
+-- and skips non-matching files.
+batchFilesMatching :: (FilePath -> CommandStart) -> Annex ()
+batchFilesMatching a = do
+	matcher <- getMatcher
+	allBatchFiles $ \f ->
+		ifM (matcher $ MatchingFile $ FileInfo f f)
+			( a f
+			, return Nothing
+			)

Command/Add.hs

@@ -62,7 +62,7 @@ seek o = allowConcurrentOutput $ do
 		Batch
 			| updateOnly o ->
 				giveup "--update --batch is not supported"
-			| otherwise -> batchFiles gofile
+			| otherwise -> batchFilesMatching gofile
 		NoBatch -> do
 			l <- workTreeItems (addThese o)
 			let go a = a gofile l

Command/Copy.hs

@@ -47,7 +47,7 @@ seek :: CopyOptions -> CommandSeek
 seek o = allowConcurrentOutput $ do
 	let go = whenAnnexed $ start o
 	case batchOption o of
-		Batch -> batchInput Right (batchCommandAction . go)
+		Batch -> batchFilesMatching go
 		NoBatch -> withKeyOptions
 			(keyOptions o) (autoMode o)
 			(Command.Move.startKey (fromToOptions o) Command.Move.RemoveNever)

Command/Drop.hs

@@ -54,7 +54,7 @@ parseDropFromOption = parseRemoteOption <$> strOption
 seek :: DropOptions -> CommandSeek
 seek o = allowConcurrentOutput $
 	case batchOption o of
-		Batch -> batchInput Right (batchCommandAction . go)
+		Batch -> batchFilesMatching go
 		NoBatch -> withKeyOptions (keyOptions o) (autoMode o)
 			(startKeys o)
 			(withFilesInGit go)

Command/Find.hs

@@ -51,7 +51,7 @@ parseFormatOption =
 seek :: FindOptions -> CommandSeek
 seek o = case batchOption o of
 	NoBatch -> withFilesInGit go =<< workTreeItems (findThese o)
-	Batch -> batchFiles go
+	Batch -> batchFilesMatching go
   where
 	go = whenAnnexed $ start o
 

Command/Get.hs

@@ -42,7 +42,7 @@ seek o = allowConcurrentOutput $ do
 	from <- maybe (pure Nothing) (Just <$$> getParsed) (getFrom o)
 	let go = whenAnnexed $ start o from
 	case batchOption o of
-		Batch -> batchInput Right (batchCommandAction . go)
+		Batch -> batchFilesMatching go
 		NoBatch -> withKeyOptions (keyOptions o) (autoMode o)
 			(startKeys from)
 			(withFilesInGit go)

Command/MetaData.hs

@@ -15,6 +15,7 @@ import Annex.WorkTree
 import Messages.JSON (JSONActionItem(..))
 import Types.Messages
 import Utility.Aeson
+import Limit
 
 import qualified Data.Set as S
 import qualified Data.Map as M
@@ -83,8 +84,11 @@ seek o = case batchOption o of
 			(seeker $ whenAnnexed $ start c o)
 			=<< workTreeItems (forFiles o)
 	Batch -> withMessageState $ \s -> case outputType s of
-		JSONOutput _ -> batchInput parseJSONInput $
+		JSONOutput _ -> ifM limited
-			commandAction . startBatch
+			( giveup "combining --batch with file matching options is not currently supported"
+			, batchInput parseJSONInput $
+				commandAction . startBatch
+			)
 		_ -> giveup "--batch is currently only supported in --json mode"
 
 start :: VectorClock -> MetaDataOptions -> FilePath -> Key -> CommandStart

Command/Move.hs

@@ -57,7 +57,7 @@ seek :: MoveOptions -> CommandSeek
 seek o = allowConcurrentOutput $ do
 	let go = whenAnnexed $ start (fromToOptions o) (removeWhen o)
 	case batchOption o of
-		Batch -> batchInput Right (batchCommandAction . go)
+		Batch -> batchFilesMatching go
 		NoBatch -> withKeyOptions (keyOptions o) False
 			(startKey (fromToOptions o) (removeWhen o))
 			(withFilesInGit go)

Command/Whereis.hs

@@ -40,7 +40,7 @@ seek o = do
 	m <- remoteMap id
 	let go = whenAnnexed $ start m
 	case batchOption o of
-		Batch -> batchFiles go
+		Batch -> batchFilesMatching go
 		NoBatch -> 
 			withKeyOptions (keyOptions o) False
 				(startKeys m)

doc/bugs/find_with_batch_does_not_apply_matching_options.mdwn

@@ -8,3 +8,6 @@ Using `git annex find --batch` with matching options seems to not apply them.
 
 ### What version of git-annex are you using? On what operating system?
 I'd rather not say ~~because it is ancient~~. Joey says this is reproducible on recent git-annex versions though.
+
+> Not only find, but a bunch of commands supporting --batch had this
+> oversight. Fixed them all. [[done]] --[[Joey]]

doc/git-annex-add.mdwn

@@ -77,8 +77,9 @@ annexed content, and other symlinks.
   the file is added, and repeat.
 
   Note that if a file is skipped (due to not existing, being gitignored,
-  already being in git etc), an empty line will be output instead of the
+  already being in git, or doesn't meet the matching options), 
-  normal output produced when adding a file.
+  an empty line will be output instead of the normal output produced
+  when adding a file.
 
 # SEE ALSO
 

doc/git-annex-copy.mdwn

@@ -80,8 +80,9 @@ Copies the content of files from or to another remote.
   are read from stdin.
 
   As each specified file is processed, the usual progress output is
-  displayed. If a file's content does not need to be copied or it
+  displayed. If a file's content does not need to be copied, or it does not
-  is not an annexed file, a blank line is output in response instead.
+  match specified matching options, or it is not an annexed file,
+  a blank line is output in response instead.
 
   Since the usual output while copying a file is verbose and not
   machine-parseable, you may want to use --json in combination with

doc/git-annex-drop.mdwn

@@ -82,6 +82,11 @@ safe to do so.
   Enables batch mode, in which lines containing names of files to drop
   are read from stdin.
 
+  As each specified file is processed, the usual output is
+  displayed. If a file's content is not present, or it does not
+  match specified matching options, or it is not an annexed file,
+  a blank line is output in response instead.
+
 * `--json`
 
   Enable JSON output. This is intended to be parsed by programs that use

doc/git-annex-get.mdwn

@@ -90,7 +90,8 @@ or transferring them from some kind of key-value store.
   are read from stdin.
 
   As each specified file is processed, the usual progress output is
-  displayed. If the specified file's content is already present, or
+  displayed. If the specified file's content is already present, 
+  or it does not match specified matching options, or
   it is not an annexed file, a blank line is output in response instead.
 
   Since the usual output while getting a file is verbose and not

doc/git-annex-metadata.mdwn

@@ -152,6 +152,9 @@ automatically.
 
 	{"file":"foo","fields":{"author":[]}}
 
+  Note that file matching options do not affect the files that are
+  processed when in batch mode.
+
 # EXAMPLES
 
 To set some tags on a file and also its author:

doc/git-annex-move.mdwn

@@ -80,7 +80,8 @@ Moves the content of files from or to another remote.
   are read from stdin.
 
   As each specified file is processed, the usual progress output is
-  displayed. If a file's content does not need to be moved or it
+  displayed. If a file's content does not need to be moved,
+  or it does not match specified matching options, or it
   is not an annexed file, a blank line is output in response instead.
 
   Since the usual output while moving a file is verbose and not

doc/git-annex-whereis.mdwn

@@ -48,7 +48,8 @@ For example:
   Enables batch mode, in which a file is read in a line from stdin,
   its information displayed, and repeat.
 
-  Note that if the file is not an annexed file, an empty line will be
+  Note that if the file is not an annexed file, or does not match
+  specified file matching options, an empty line will be
   output instead.
 
 * `--json`