From ab7b5a492cfa071c50d410e6c9c894b0ed4cf62e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 25 Aug 2021 14:20:33 -0400 Subject: [PATCH] --batch-keys New --batch-keys option added to these commands: get, drop, move, copy, whereis git-annex-matching-options had to be reworded since some of its options can be used to match on keys, not only files. Sponsored-by: Luke Shumaker on Patreon --- CHANGELOG | 2 + CmdLine/Batch.hs | 139 +++++++++++++----- Command/Add.hs | 4 +- Command/AddUrl.hs | 2 +- Command/CheckPresentKey.hs | 2 +- Command/Copy.hs | 7 +- Command/Drop.hs | 4 +- Command/DropKey.hs | 2 +- Command/Find.hs | 4 +- Command/FromKey.hs | 4 +- Command/Get.hs | 4 +- Command/Info.hs | 2 +- Command/MetaData.hs | 2 +- Command/Move.hs | 7 +- Command/ReKey.hs | 2 +- Command/RegisterUrl.hs | 16 +- Command/RmUrl.hs | 2 +- Command/SetPresentKey.hs | 2 +- Command/UnregisterUrl.hs | 2 +- Command/Whereis.hs | 4 +- doc/git-annex-copy.mdwn | 11 +- doc/git-annex-drop.mdwn | 15 +- doc/git-annex-find.mdwn | 4 + doc/git-annex-fsck.mdwn | 4 +- doc/git-annex-get.mdwn | 10 +- doc/git-annex-log.mdwn | 4 +- doc/git-annex-matching-options.mdwn | 61 ++++---- doc/git-annex-metadata.mdwn | 6 +- doc/git-annex-mirror.mdwn | 4 +- doc/git-annex-move.mdwn | 11 +- doc/git-annex-whereis.mdwn | 12 +- doc/todo/git-annex-get_--batch_--key.mdwn | 2 +- ..._c778976121cff4e1fbec3acd0cc70648._comment | 20 +++ 33 files changed, 244 insertions(+), 133 deletions(-) create mode 100644 doc/todo/git-annex-get_--batch_--key/comment_5_c778976121cff4e1fbec3acd0cc70648._comment diff --git a/CHANGELOG b/CHANGELOG index 9b7d4cf377..c59603ba6b 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -15,6 +15,8 @@ git-annex (8.20210804) UNRELEASED; urgency=medium * unused: Skip the refs/annex/last-index ref that git-annex recently started creating. * Fix test suite failure on Windows. + * New --batch-keys option added to these commands: + get, drop, move, copy, whereis -- Joey Hess Tue, 03 Aug 2021 12:22:45 -0400 diff --git a/CmdLine/Batch.hs b/CmdLine/Batch.hs index 799f586e84..15fd89710e 100644 --- a/CmdLine/Batch.hs +++ b/CmdLine/Batch.hs @@ -1,6 +1,6 @@ {- git-annex batch commands - - - Copyright 2015-2020 Joey Hess + - Copyright 2015-2021 Joey Hess - - Licensed under the GNU AGPL version 3 or higher. -} @@ -24,22 +24,42 @@ import Types.Concurrency data BatchMode = Batch BatchFormat | NoBatch -data BatchFormat = BatchLine | BatchNull +data BatchFormat = BatchFormat BatchSeparator BatchKeys -parseBatchOption :: Parser BatchMode -parseBatchOption = go +data BatchSeparator = BatchLine | BatchNull + +newtype BatchKeys = BatchKeys Bool + +parseBatchOption :: Bool -> Parser BatchMode +parseBatchOption supportbatchkeysoption = go <$> switch ( long "batch" - <> help "enable batch mode" + <> help batchhelp ) - <*> switch + <*> batchkeysswitch + <*> flag BatchLine BatchNull ( short 'z' <> help "null delimited batch input" ) where - go True False = Batch BatchLine - go True True = Batch BatchNull - go False _ = NoBatch + go True False batchseparator = + Batch (BatchFormat batchseparator (BatchKeys False)) + go _ True batchseparator = + Batch (BatchFormat batchseparator (BatchKeys True)) + go _ _ _ = NoBatch + + batchhelp = "enable batch mode" ++ + if supportbatchkeysoption + then ", with files input" + else "" + batchkeyshelp = "enable batch mode, with keys input" + + batchkeysswitch + | supportbatchkeysoption = switch + ( long "batch-keys" + <> help batchkeyshelp + ) + | otherwise = pure False -- A batchable command can run in batch mode, or not. -- In batch mode, one line at a time is read, parsed, and a reply output to @@ -52,7 +72,7 @@ batchable handler parser paramdesc = batchseeker <$> batchparser where batchparser = (,,) <$> parser - <*> parseBatchOption + <*> parseBatchOption False <*> cmdParams paramdesc batchseeker (opts, NoBatch, params) = @@ -68,7 +88,7 @@ batchable handler parser paramdesc = batchseeker <$> batchparser -- mode, exit on bad input. batchBadInput :: BatchMode -> Annex () batchBadInput NoBatch = liftIO exitFailure -batchBadInput (Batch _) = liftIO $ putStrLn "" +batchBadInput _ = liftIO $ putStrLn "" -- Reads lines of batch mode input, runs a parser, and passes the result -- to the action. @@ -87,12 +107,12 @@ batchInput fmt parser a = go =<< batchLines fmt parseerr s = giveup $ "Batch input parse failure: " ++ s batchLines :: BatchFormat -> Annex [String] -batchLines fmt = do +batchLines (BatchFormat sep _) = do checkBatchConcurrency enableInteractiveBranchAccess liftIO $ splitter <$> getContents where - splitter = case fmt of + splitter = case sep of BatchLine -> lines BatchNull -> splitc '\0' @@ -116,37 +136,76 @@ batchCommandStart :: CommandStart -> CommandStart batchCommandStart a = a >>= \case Just v -> return (Just v) Nothing -> do - batchBadInput (Batch BatchLine) + batchBadInput (Batch (BatchFormat BatchLine (BatchKeys False))) return Nothing -- Reads lines of batch input and passes the filepaths to a CommandStart -- to handle them. -- --- Absolute filepaths are converted to relative, because in non-batch --- mode, that is done when CmdLine.Seek uses git ls-files. --- -- File matching options are checked, and non-matching files skipped. -batchFilesMatching :: BatchFormat -> ((SeekInput, RawFilePath) -> CommandStart) -> Annex () -batchFilesMatching fmt a = do - matcher <- getMatcher - go $ \si f -> - let f' = toRawFilePath f - in ifM (matcher $ MatchingFile $ FileInfo f' f' Nothing) - ( a (si, f') - , return Nothing - ) - where - go a' = batchInput fmt - (Right . fromRawFilePath <$$> liftIO . relPathCwdToFile . toRawFilePath) - (batchCommandAction . uncurry a') +batchFiles :: BatchFormat -> ((SeekInput, RawFilePath) -> CommandStart) -> Annex () +batchFiles fmt a = batchFilesKeys fmt $ \(si, v) -> case v of + Right f -> a (si, f) + Left _k -> return Nothing -batchAnnexedFilesMatching :: BatchFormat -> AnnexedFileSeeker -> Annex () -batchAnnexedFilesMatching fmt seeker = batchFilesMatching fmt $ \(si, bf) -> - flip whenAnnexed bf $ \f k -> - case checkContentPresent seeker of - Just v -> do - present <- inAnnex k - if present == v - then startAction seeker si f k - else return Nothing - Nothing -> startAction seeker si f k +batchFilesKeys :: BatchFormat -> ((SeekInput, Either Key RawFilePath) -> CommandStart) -> Annex () +batchFilesKeys fmt a = do + matcher <- getMatcher + go $ \si v -> case v of + Right f -> + let f' = toRawFilePath f + in ifM (matcher $ MatchingFile $ FileInfo f' f' Nothing) + ( a (si, Right f') + , return Nothing + ) + Left k -> a (si, Left k) + where + go a' = batchInput fmt parser (batchCommandAction . uncurry a') + parser = case fmt of + -- Absolute filepaths are converted to relative, + -- because in non-batch mode, that is done when + -- CmdLine.Seek uses git ls-files. + BatchFormat _ (BatchKeys False) -> + Right . Right . fromRawFilePath + <$$> liftIO . relPathCwdToFile . toRawFilePath + BatchFormat _ (BatchKeys True) -> \i -> + pure $ case deserializeKey i of + Just k -> Right (Left k) + Nothing -> Left "not a valid key" + +batchAnnexedFiles :: BatchFormat -> AnnexedFileSeeker -> Annex () +batchAnnexedFiles fmt seeker = batchAnnexed fmt seeker (const (return Nothing)) + +-- Reads lines of batch input and passes filepaths to the AnnexedFileSeeker +-- to handle them. Or, with --batch-keys, passes keys to the keyaction. +-- +-- Matching options are checked, and non-matching items skipped. +batchAnnexed :: BatchFormat -> AnnexedFileSeeker -> ((SeekInput, Key, ActionItem) -> CommandStart) -> Annex () +batchAnnexed fmt seeker keyaction = do + matcher <- getMatcher + batchFilesKeys fmt $ \(si, v) -> + case v of + Right bf -> flip whenAnnexed bf $ \f k -> + checkpresent k $ + startAction seeker si f k + Left k -> ifM (matcher (MatchingInfo (mkinfo k))) + ( checkpresent k $ + keyaction (si, k, mkActionItem k) + , return Nothing) + where + checkpresent k cont = case checkContentPresent seeker of + Just v -> do + present <- inAnnex k + if present == v + then cont + else return Nothing + Nothing -> cont + + mkinfo k = ProvidedInfo + { providedFilePath = Nothing + , providedKey = Just k + , providedFileSize = Nothing + , providedMimeType = Nothing + , providedMimeEncoding = Nothing + , providedLinkType = Nothing + } diff --git a/Command/Add.hs b/Command/Add.hs index 9398865f74..56495825da 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -50,7 +50,7 @@ data AddOptions = AddOptions optParser :: CmdParamsDesc -> Parser AddOptions optParser desc = AddOptions <$> cmdParams desc - <*> parseBatchOption + <*> parseBatchOption False <*> switch ( long "update" <> short 'u' @@ -95,7 +95,7 @@ seek o = startConcurrency commandStages $ do Batch fmt | updateOnly o -> giveup "--update --batch is not supported" - | otherwise -> batchFilesMatching fmt gofile + | otherwise -> batchFiles fmt gofile NoBatch -> do -- Avoid git ls-files complaining about files that -- are not known to git yet, since this will add diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs index 70cc3239f9..eb99119ac4 100644 --- a/Command/AddUrl.hs +++ b/Command/AddUrl.hs @@ -76,7 +76,7 @@ optParser desc = AddUrlOptions <> help "add a suffix to the filename" )) <*> parseDownloadOptions True - <*> parseBatchOption + <*> parseBatchOption False <*> switch ( long "with-files" <> help "parse batch mode lines of the form \"$url $file\"" diff --git a/Command/CheckPresentKey.hs b/Command/CheckPresentKey.hs index 1f6a554e85..46211523bf 100644 --- a/Command/CheckPresentKey.hs +++ b/Command/CheckPresentKey.hs @@ -26,7 +26,7 @@ data CheckPresentKeyOptions = CheckPresentKeyOptions optParser :: CmdParamsDesc -> Parser CheckPresentKeyOptions optParser desc = CheckPresentKeyOptions <$> cmdParams desc - <*> parseBatchOption + <*> parseBatchOption False seek :: CheckPresentKeyOptions -> CommandSeek seek o = case batchOption o of diff --git a/Command/Copy.hs b/Command/Copy.hs index 15f2ea4f16..a6ddf4f80f 100644 --- a/Command/Copy.hs +++ b/Command/Copy.hs @@ -33,7 +33,7 @@ optParser desc = CopyOptions <*> parseFromToHereOptions <*> optional (parseKeyOptions <|> parseFailedTransfersOption) <*> parseAutoOption - <*> parseBatchOption + <*> parseBatchOption True instance DeferredParseClass CopyOptions where finishParse v = CopyOptions @@ -48,10 +48,10 @@ seek o = startConcurrency commandStages $ do case batchOption o of NoBatch -> withKeyOptions (keyOptions o) (autoMode o) seeker - (commandAction . Command.Move.startKey (fromToOptions o) Command.Move.RemoveNever) + (commandAction . keyaction) (withFilesInGitAnnex ww seeker) =<< workTreeItems ww (copyFiles o) - Batch fmt -> batchAnnexedFilesMatching fmt seeker + Batch fmt -> batchAnnexed fmt seeker keyaction where ww = WarnUnmatchLsFiles @@ -63,6 +63,7 @@ seek o = startConcurrency commandStages $ do Left ToHere -> Just False , usesLocationLog = True } + keyaction = Command.Move.startKey (fromToOptions o) Command.Move.RemoveNever {- A copy is just a move that does not delete the source file. - However, auto mode avoids unnecessary copies, and avoids getting or diff --git a/Command/Drop.hs b/Command/Drop.hs index b8c930f6f3..a9a4314cac 100644 --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -41,7 +41,7 @@ optParser desc = DropOptions <*> optional parseDropFromOption <*> parseAutoOption <*> optional parseKeyOptions - <*> parseBatchOption + <*> parseBatchOption True parseDropFromOption :: Parser (DeferredParse Remote) parseDropFromOption = parseRemoteOption <$> strOption @@ -67,11 +67,11 @@ seek o = startConcurrency commandStages $ do , usesLocationLog = True } case batchOption o of - Batch fmt -> batchAnnexedFilesMatching fmt seeker NoBatch -> withKeyOptions (keyOptions o) (autoMode o) seeker (commandAction . startKeys o from) (withFilesInGitAnnex ww seeker) =<< workTreeItems ww (dropFiles o) + Batch fmt -> batchAnnexed fmt seeker (startKeys o from) where ww = WarnUnmatchLsFiles diff --git a/Command/DropKey.hs b/Command/DropKey.hs index ddc2bea5f6..580c6f1c22 100644 --- a/Command/DropKey.hs +++ b/Command/DropKey.hs @@ -27,7 +27,7 @@ data DropKeyOptions = DropKeyOptions optParser :: CmdParamsDesc -> Parser DropKeyOptions optParser desc = DropKeyOptions <$> cmdParams desc - <*> parseBatchOption + <*> parseBatchOption False seek :: DropKeyOptions -> CommandSeek seek o = do diff --git a/Command/Find.hs b/Command/Find.hs index 96f1f3b8bc..d89ff2b967 100644 --- a/Command/Find.hs +++ b/Command/Find.hs @@ -39,7 +39,7 @@ optParser desc = FindOptions <$> cmdParams desc <*> optional parseFormatOption <*> optional parseBranchKeysOption - <*> parseBatchOption + <*> parseBatchOption False parseFormatOption :: Parser Utility.Format.Format parseFormatOption = @@ -69,7 +69,7 @@ seek o = do (commandAction . startKeys o) (withFilesInGitAnnex ww seeker) =<< workTreeItems ww (findThese o) - Batch fmt -> batchAnnexedFilesMatching fmt seeker + Batch fmt -> batchAnnexedFiles fmt seeker where ww = WarnUnmatchLsFiles diff --git a/Command/FromKey.hs b/Command/FromKey.hs index 16ff1693f4..e4591a184f 100644 --- a/Command/FromKey.hs +++ b/Command/FromKey.hs @@ -35,7 +35,7 @@ data FromKeyOptions = FromKeyOptions optParser :: CmdParamsDesc -> Parser FromKeyOptions optParser desc = FromKeyOptions <$> cmdParams desc - <*> parseBatchOption + <*> parseBatchOption False seek :: FromKeyOptions -> CommandSeek seek o = do @@ -43,7 +43,7 @@ seek o = do case (batchOption o, keyFilePairs o) of (Batch fmt, _) -> seekBatch matcher fmt -- older way of enabling batch input, does not support BatchNull - (NoBatch, []) -> seekBatch matcher BatchLine + (NoBatch, []) -> seekBatch matcher (BatchFormat BatchLine (BatchKeys False)) (NoBatch, ps) -> do force <- Annex.getState Annex.force withPairs (commandAction . start matcher force) ps diff --git a/Command/Get.hs b/Command/Get.hs index f443c86bf3..eef40c4eb3 100644 --- a/Command/Get.hs +++ b/Command/Get.hs @@ -34,7 +34,7 @@ optParser desc = GetOptions <*> optional (parseRemoteOption <$> parseFromOption) <*> parseAutoOption <*> optional (parseIncompleteOption <|> parseKeyOptions <|> parseFailedTransfersOption) - <*> parseBatchOption + <*> parseBatchOption True seek :: GetOptions -> CommandSeek seek o = startConcurrency downloadStages $ do @@ -49,7 +49,7 @@ seek o = startConcurrency downloadStages $ do (commandAction . startKeys from) (withFilesInGitAnnex ww seeker) =<< workTreeItems ww (getFiles o) - Batch fmt -> batchAnnexedFilesMatching fmt seeker + Batch fmt -> batchAnnexed fmt seeker (startKeys from) where ww = WarnUnmatchLsFiles diff --git a/Command/Info.hs b/Command/Info.hs index 2d97603e9b..f4ef0df800 100644 --- a/Command/Info.hs +++ b/Command/Info.hs @@ -114,7 +114,7 @@ optParser desc = InfoOptions ( long "bytes" <> help "display file sizes in bytes" ) - <*> parseBatchOption + <*> parseBatchOption False seek :: InfoOptions -> CommandSeek seek o = case batchOption o of diff --git a/Command/MetaData.hs b/Command/MetaData.hs index 56c25df6ca..f2fd87ffbf 100644 --- a/Command/MetaData.hs +++ b/Command/MetaData.hs @@ -44,7 +44,7 @@ optParser desc = MetaDataOptions <$> cmdParams desc <*> ((Get <$> getopt) <|> (Set <$> some modopts) <|> pure GetAll) <*> optional parseKeyOptions - <*> parseBatchOption + <*> parseBatchOption False where getopt = option (eitherReader (mkMetaField . T.pack)) ( long "get" <> short 'g' <> metavar paramField diff --git a/Command/Move.hs b/Command/Move.hs index bff099dd2c..781bdb6138 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -44,7 +44,7 @@ optParser desc = MoveOptions <*> parseFromToHereOptions <*> pure RemoveSafe <*> optional (parseKeyOptions <|> parseFailedTransfersOption) - <*> parseBatchOption + <*> parseBatchOption True instance DeferredParseClass MoveOptions where finishParse v = MoveOptions @@ -61,10 +61,10 @@ seek :: MoveOptions -> CommandSeek seek o = startConcurrency stages $ do case batchOption o of NoBatch -> withKeyOptions (keyOptions o) False seeker - (commandAction . startKey (fromToOptions o) (removeWhen o)) + (commandAction . keyaction) (withFilesInGitAnnex ww seeker) =<< workTreeItems ww (moveFiles o) - Batch fmt -> batchAnnexedFilesMatching fmt seeker + Batch fmt -> batchAnnexed fmt seeker keyaction where seeker = AnnexedFileSeeker { startAction = start (fromToOptions o) (removeWhen o) @@ -78,6 +78,7 @@ seek o = startConcurrency stages $ do Right (FromRemote _) -> downloadStages Right (ToRemote _) -> commandStages Left ToHere -> downloadStages + keyaction = startKey (fromToOptions o) (removeWhen o) ww = WarnUnmatchLsFiles start :: FromToHereOptions -> RemoveWhen -> SeekInput -> RawFilePath -> Key -> CommandStart diff --git a/Command/ReKey.hs b/Command/ReKey.hs index 077dd5a628..ab2f9fa241 100644 --- a/Command/ReKey.hs +++ b/Command/ReKey.hs @@ -33,7 +33,7 @@ data ReKeyOptions = ReKeyOptions optParser :: CmdParamsDesc -> Parser ReKeyOptions optParser desc = ReKeyOptions <$> cmdParams desc - <*> parseBatchOption + <*> parseBatchOption False -- Split on the last space, since a FilePath can contain whitespace, -- but a Key very rarely does. diff --git a/Command/RegisterUrl.hs b/Command/RegisterUrl.hs index 583b170143..1ec953d02b 100644 --- a/Command/RegisterUrl.hs +++ b/Command/RegisterUrl.hs @@ -28,11 +28,12 @@ data RegisterUrlOptions = RegisterUrlOptions optParser :: CmdParamsDesc -> Parser RegisterUrlOptions optParser desc = RegisterUrlOptions <$> cmdParams desc - <*> parseBatchOption + <*> parseBatchOption False seek :: RegisterUrlOptions -> CommandSeek seek o = case (batchOption o, keyUrlPairs o) of - (Batch fmt, _) -> commandAction $ startMass setUrlPresent fmt + (Batch (BatchFormat sep _), _) -> + commandAction $ startMass setUrlPresent sep -- older way of enabling batch input, does not support BatchNull (NoBatch, []) -> commandAction $ startMass setUrlPresent BatchLine (NoBatch, ps) -> withWords (commandAction . start setUrlPresent) ps @@ -46,14 +47,15 @@ start a (keyname:url:[]) = si = SeekInput [keyname, url] start _ _ = giveup "specify a key and an url" -startMass :: (Key -> URLString -> Annex ()) -> BatchFormat -> CommandStart -startMass a fmt = +startMass :: (Key -> URLString -> Annex ()) -> BatchSeparator -> CommandStart +startMass a sep = starting "registerurl" (ActionItemOther (Just "stdin")) (SeekInput []) $ - performMass a fmt + performMass a sep -performMass :: (Key -> URLString -> Annex ()) -> BatchFormat -> CommandPerform -performMass a fmt = go True =<< map (separate (== ' ')) <$> batchLines fmt +performMass :: (Key -> URLString -> Annex ()) -> BatchSeparator -> CommandPerform +performMass a sep = go True =<< map (separate (== ' ')) <$> batchLines fmt where + fmt = BatchFormat sep (BatchKeys False) go status [] = next $ return status go status ((keyname,u):rest) | not (null keyname) && not (null u) = do let key = keyOpt keyname diff --git a/Command/RmUrl.hs b/Command/RmUrl.hs index 83092800f3..bd854f64a0 100644 --- a/Command/RmUrl.hs +++ b/Command/RmUrl.hs @@ -25,7 +25,7 @@ data RmUrlOptions = RmUrlOptions optParser :: CmdParamsDesc -> Parser RmUrlOptions optParser desc = RmUrlOptions <$> cmdParams desc - <*> parseBatchOption + <*> parseBatchOption False seek :: RmUrlOptions -> CommandSeek seek o = case batchOption o of diff --git a/Command/SetPresentKey.hs b/Command/SetPresentKey.hs index e90f389348..50749886e6 100644 --- a/Command/SetPresentKey.hs +++ b/Command/SetPresentKey.hs @@ -26,7 +26,7 @@ data SetPresentKeyOptions = SetPresentKeyOptions optParser :: CmdParamsDesc -> Parser SetPresentKeyOptions optParser desc = SetPresentKeyOptions <$> cmdParams desc - <*> parseBatchOption + <*> parseBatchOption False seek :: SetPresentKeyOptions -> CommandSeek seek o = case batchOption o of diff --git a/Command/UnregisterUrl.hs b/Command/UnregisterUrl.hs index 7d4d65beb7..2e2dcc83ae 100644 --- a/Command/UnregisterUrl.hs +++ b/Command/UnregisterUrl.hs @@ -21,7 +21,7 @@ cmd = command "unregisterurl" seek :: RegisterUrlOptions -> CommandSeek seek o = case (batchOption o, keyUrlPairs o) of - (Batch fmt, _) -> commandAction $ startMass unregisterUrl fmt + (Batch (BatchFormat sep _), _) -> commandAction $ startMass unregisterUrl sep (NoBatch, ps) -> withWords (commandAction . start unregisterUrl) ps unregisterUrl :: Key -> String -> Annex () diff --git a/Command/Whereis.hs b/Command/Whereis.hs index 81c297aad1..58f26db1f1 100644 --- a/Command/Whereis.hs +++ b/Command/Whereis.hs @@ -39,7 +39,7 @@ optParser :: CmdParamsDesc -> Parser WhereisOptions optParser desc = WhereisOptions <$> cmdParams desc <*> optional parseKeyOptions - <*> parseBatchOption + <*> parseBatchOption True <*> optional parseFormatOption parseFormatOption :: Parser Utility.Format.Format @@ -62,7 +62,7 @@ seek o = do (commandAction . startKeys o m) (withFilesInGitAnnex ww seeker) =<< workTreeItems ww (whereisFiles o) - Batch fmt -> batchAnnexedFilesMatching fmt seeker + Batch fmt -> batchAnnexed fmt seeker (startKeys o m) where ww = WarnUnmatchLsFiles diff --git a/doc/git-annex-copy.mdwn b/doc/git-annex-copy.mdwn index e77d9a508b..5a4e805cdf 100644 --- a/doc/git-annex-copy.mdwn +++ b/doc/git-annex-copy.mdwn @@ -74,10 +74,10 @@ Paths of files or directories to operate on can be specified. Use this option to copy a specified key. -* file matching options +* matching options The [[git-annex-matching-options]](1) - can be used to specify files to copy. + can be used to specify what to copy. * `--batch` @@ -93,10 +93,13 @@ Paths of files or directories to operate on can be specified. machine-parseable, you may want to use --json in combination with --batch. +* `--batch-keys` + + This is like `--batch` but the lines read from stdin are parsed as keys. + * `-z` - Makes the `--batch` input be delimited by nulls instead of the usual - newlines. + Makes batch input be delimited by nulls instead of the usual newlines. * `--json` diff --git a/doc/git-annex-drop.mdwn b/doc/git-annex-drop.mdwn index 23f0b70268..482aa9906c 100644 --- a/doc/git-annex-drop.mdwn +++ b/doc/git-annex-drop.mdwn @@ -87,10 +87,10 @@ Paths of files or directories to drop can be specified. Note that this bypasses checking the .gitattributes annex.numcopies setting and required content settings. -* file matching options +* matching options The [[git-annex-matching-options]](1) - can be used to specify files to drop. + can be used to specify what to drop. * `--jobs=N` `-JN` @@ -110,10 +110,17 @@ Paths of files or directories to drop can be specified. match specified matching options, or it is not an annexed file, a blank line is output in response instead. +* `--batch-keys` + + This is like `--batch` but the lines read from stdin are parsed as keys. + + Note that this bypasses checking the .gitattributes annex.numcopies + setting and required content settings. + * `-z` - Makes the `--batch` input be delimited by nulls instead of the usual - newlines. + Makes the batch input be delimited by nulls + instead of the usual newlines. * `--json` diff --git a/doc/git-annex-find.mdwn b/doc/git-annex-find.mdwn index 2decd932b8..e8b5feead2 100644 --- a/doc/git-annex-find.mdwn +++ b/doc/git-annex-find.mdwn @@ -72,6 +72,10 @@ finds files in the current directory and its subdirectories. or otherwise doesn't meet the matching options, an empty line will be output instead. +* `--batch-keys` + + This is like `--batch` but the lines read from stdin are parsed as keys. + * `-z` Makes the `--batch` input be delimited by nulls instead of the usual diff --git a/doc/git-annex-fsck.mdwn b/doc/git-annex-fsck.mdwn index edc0360487..faa58aaa1c 100644 --- a/doc/git-annex-fsck.mdwn +++ b/doc/git-annex-fsck.mdwn @@ -89,10 +89,10 @@ better format. Use this option to fsck a specified key. -* file matching options +* matching options The [[git-annex-matching-options]](1) - can be used to specify files to fsck. + can be used to control what to fsck. * `--jobs=N` `-JN` diff --git a/doc/git-annex-get.mdwn b/doc/git-annex-get.mdwn index 44f4fba0ca..8fe1ab6f3e 100644 --- a/doc/git-annex-get.mdwn +++ b/doc/git-annex-get.mdwn @@ -56,10 +56,10 @@ be specified. parallel. (Remotes with lower costs are still preferred over higher cost remotes.) -* file matching options +* matching options The [[git-annex-matching-options]](1) - can be used to specify files to get. + can be used to control what to get. * `--incomplete` @@ -114,9 +114,13 @@ be specified. machine-parseable, you may want to use --json in combination with --batch. +* `--batch-keys` + + This is like `--batch` but the lines read from stdin are parsed as keys. + * `-z` - Makes the `--batch` input be delimited by nulls instead of the usual + Makes batch input be delimited by nulls instead of the usual newlines. * `--json` diff --git a/doc/git-annex-log.mdwn b/doc/git-annex-log.mdwn index 053b761ca4..68cf79fb18 100644 --- a/doc/git-annex-log.mdwn +++ b/doc/git-annex-log.mdwn @@ -39,10 +39,10 @@ false, information may not have been committed to the branch yet. Generates output suitable for the `gource` visualization program. -* file matching options +* matching options The [[git-annex-matching-options]](1) - can be used to specify files to act on. + can be used to control what to act on. * `--all` `-A` diff --git a/doc/git-annex-matching-options.mdwn b/doc/git-annex-matching-options.mdwn index a2670eb34b..93172165ca 100644 --- a/doc/git-annex-matching-options.mdwn +++ b/doc/git-annex-matching-options.mdwn @@ -1,11 +1,12 @@ # NAME -git-annex-matching-options - specifying files to act on +git-annex-matching-options - specifying what to act on # DESCRIPTION Many git-annex commands support using these options to specify which -files they act on. +files they act on. Some of these options can also be used by commands to +specify which keys they act on. Arbitrarily complicated expressions can be built using these options. For example: @@ -58,8 +59,8 @@ in either of two repositories. * `--in=repository` - Matches only files that git-annex believes have their contents present - in a repository. Note that it does not check the repository to verify + Matches only when git-annex believes that the content is present in a + repository. Note that it does not check the repository to verify that it still has the content. The repository should be specified using the name of a configured remote, @@ -68,8 +69,8 @@ in either of two repositories. * `--in=repository@{date}` - Matches files currently in the work tree whose content was present in - the repository on the given date. + Matches only when the content was present in a repository on the given + date. The date is specified in the same syntax documented in gitrevisions(7). Note that this uses the reflog, so dates far in the @@ -81,13 +82,13 @@ in either of two repositories. * `--copies=number` - Matches only files that git-annex believes to have the specified number + Matches only when git-annex believes there are the specified number of copies, or more. Note that it does not check remotes to verify that the copies still exist. * `--copies=trustlevel:number` - Matches only files that git-annex believes have the specified number of + Matches only when git-annex believes there are the specified number of copies, on remotes with the specified trust level. For example, `--copies=trusted:2` @@ -96,14 +97,14 @@ in either of two repositories. * `--copies=groupname:number` - Matches only files that git-annex believes have the specified number of + Matches only when git-annex believes there are the specified number of copies, on remotes in the specified group. For example, `--copies=archive:2` * `--lackingcopies=number` - Matches only files that git-annex believes need the specified number or - more additional copies to be made in order to satisfy their numcopies + Matches only when git-annex beleives that the specified number or + more additional copies to be made in order to satisfy numcopies settings. * `--approxlackingcopies=number` @@ -113,23 +114,23 @@ in either of two repositories. * `--inbackend=name` - Matches only files whose content is stored using the specified key-value + Matches only when content is stored using the specified key-value backend. * `--securehash` - Matches only files whose content is hashed using a cryptographically + Matches only when content is hashed using a cryptographically secure function. * `--inallgroup=groupname` - Matches only files that git-annex believes are present in all repositories - in the specified group. + Matches only when git-annex believes content is present in + all repositories in the specified group. * `--smallerthan=size` * `--largerthan=size` - Matches only files whose content is smaller than, or larger than the + Matches only when the content is is smaller than, or larger than the specified size. The size can be specified with any commonly used units, for example, @@ -137,14 +138,14 @@ in either of two repositories. * `--metadata field=glob` - Matches only files that have a metadata field attached with a value that + Matches only when there is a metadata field attached with a value that matches the glob. The values of metadata fields are matched case insensitively. * `--metadata fieldnumber` * `--metadata field<=number` / `--metadata field>=number` - Matches only files that have a metadata field attached with a value that + Matches only when there is a metadata field attached with a value that is a number and is less than or greater than the specified number. (Note that you will need to quote the second parameter to avoid @@ -152,34 +153,34 @@ in either of two repositories. * `--want-get` - Matches files that the preferred content settings for the repository - make it want to get. Note that this will match even files that are - already present, unless limited with e.g., `--not --in .` + Matches only when the preferred content settings for the repository + make it want to get content. Note that this will match even when + the content is already present, unless limited with e.g., `--not --in .` * `--want-drop` - Matches files that the preferred content settings for the repository - make it want to drop. Note that this will match even files that have - already been dropped, unless limited with e.g., `--in .` + Matches only when the preferred content settings for the repository + make it want to drop content. Note that this will match even when + the content is not present, unless limited with e.g., `--in .` - Files that this matches will not necessarily be dropped by + Things that this matches will not necessarily be dropped by `git-annex drop --auto`. This does not check that there are enough copies to drop. Also the same content may be used by a file that is not wanted to be dropped. * `--accessedwithin=interval` - Matches files that were accessed recently, within the specified time + Matches when the content was accessed recently, within the specified time interval. The interval can be in the form "5m" or "1h" or "2d" or "1y", or a combination such as "1h5m". - So for example, `--accessedwithin=1d` matches files that have been + So for example, `--accessedwithin=1d` matches when the content was accessed within the past day. If the OS or filesystem does not support access times, this will not - match any files. + match anything. * `--unlocked` @@ -220,8 +221,8 @@ in either of two repositories. * `--not` - Inverts the next matching option. For example, to only act on - files with less than 3 copies, use `--not --copies=3` + Inverts the next matching option. For example, to match + when there are less than 3 copies, use `--not --copies=3` * `--and` diff --git a/doc/git-annex-metadata.mdwn b/doc/git-annex-metadata.mdwn index 811b621128..e5b8b1948c 100644 --- a/doc/git-annex-metadata.mdwn +++ b/doc/git-annex-metadata.mdwn @@ -82,10 +82,10 @@ the modified file. throughout the files in a directory. This option enables such recursive setting. -* file matching options +* matching options The [[git-annex-matching-options]](1) - can be used to specify files to act on. + can be used to control what to act on. * `--all` `-A` @@ -159,7 +159,7 @@ the modified file. {"file":"foo","fields":{"author":[]}} - Note that file matching options do not affect the files that are + Note that matching options do not affect the files that are processed when in batch mode. * Also the [[git-annex-common-options]](1) can be used. diff --git a/doc/git-annex-mirror.mdwn b/doc/git-annex-mirror.mdwn index b125050e3f..688d0c3f08 100644 --- a/doc/git-annex-mirror.mdwn +++ b/doc/git-annex-mirror.mdwn @@ -63,10 +63,10 @@ contents. Use [[git-annex-sync]](1) for that. Operate on files that have recently failed to be transferred. -* file matching options +* matching options The [[git-annex-matching-options]](1) - can be used to specify files to mirror. + can be used to control what to mirror. * `--json` diff --git a/doc/git-annex-move.mdwn b/doc/git-annex-move.mdwn index 5eeb7a3e26..0fbea84d58 100644 --- a/doc/git-annex-move.mdwn +++ b/doc/git-annex-move.mdwn @@ -74,10 +74,10 @@ Paths of files or directories to operate on can be specified. already has content. This can be faster, but might skip moving content to the remote in some cases. -* file matching options +* matching options The [[git-annex-matching-options]](1) - can be used to specify files to move. + can be used to control what to move. * `--batch` @@ -93,10 +93,13 @@ Paths of files or directories to operate on can be specified. machine-parseable, you may want to use --json in combination with --batch. +* `--batch-keys` + + This is like `--batch` but the lines read from stdin are parsed as keys. + * `-z` - Makes the `--batch` input be delimited by nulls instead of the usual - newlines. + Makes batch input be delimited by nulls instead of the usual newlines. * `--json` diff --git a/doc/git-annex-whereis.mdwn b/doc/git-annex-whereis.mdwn index ee6bd89d7b..032a9ee02b 100644 --- a/doc/git-annex-whereis.mdwn +++ b/doc/git-annex-whereis.mdwn @@ -26,10 +26,10 @@ received from remotes. # OPTIONS -* file matching options +* matching options The [[git-annex-matching-options]](1) - can be used to specify files to act on. + can be used to control what to act on. * `--key=keyname` @@ -53,12 +53,16 @@ received from remotes. its information displayed, and repeat. Note that if the file is not an annexed file, or does not match - specified file matching options, an empty line will be + specified matching options, an empty line will be output instead. +* `--batch-keys` + + This is like `--batch` but the lines read from stdin are parsed as keys. + * `-z` - Makes the `--batch` input be delimited by nulls instead of the usual + Makes batch input be delimited by nulls instead of the usual newlines. * `--json` diff --git a/doc/todo/git-annex-get_--batch_--key.mdwn b/doc/todo/git-annex-get_--batch_--key.mdwn index a54d539102..fd4966a0a1 100644 --- a/doc/todo/git-annex-get_--batch_--key.mdwn +++ b/doc/todo/git-annex-get_--batch_--key.mdwn @@ -1,3 +1,3 @@ Can git-annex-get be extended so that "git-annex-get --batch --key" fetches the keys (rather than filenames) given in the input? -[[!tag needsthought]] +> [[done]] --[[Joey]] diff --git a/doc/todo/git-annex-get_--batch_--key/comment_5_c778976121cff4e1fbec3acd0cc70648._comment b/doc/todo/git-annex-get_--batch_--key/comment_5_c778976121cff4e1fbec3acd0cc70648._comment new file mode 100644 index 0000000000..43719f5b29 --- /dev/null +++ b/doc/todo/git-annex-get_--batch_--key/comment_5_c778976121cff4e1fbec3acd0cc70648._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 5""" + date="2021-08-25T18:06:29Z" + content=""" +I've implemented --batch-keys for the commands: get, drop, move, copy, whereis + +That covers everything mentioned here except for dead, but that does not +support --batch yet, so if batch mode is needed for it, it can just use +--batch, not --batch-keys. However, after a recent change that makes +dropping unused keys automatically mark them dead, I suspect there +will not be a use case for that. + +Most of the other commands that use --batch don't make sense to support +--batch-keys. Eg, add and find can't operate on keys, while +fromkey already operates on keys. About the only one that might is +rmurl, but it uses a custom batch format so would not be able to use the +current --batch-keys implementation. If someone needs that or some other +one, they can open a new todo. +"""]]