From a6c1d9752b707fedb9bd4cf743742d1bf16538e7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 18 Jan 2023 14:42:39 -0400 Subject: [PATCH] move/copy: option parsing for --from with --to Allowing --from and --to as an alternative to --from or --to is hard to do with optparse-applicative! The obvious approach of (pfrom <|> pto <|> pfromandto) does not work when pfromandto uses the same option names as pfrom and pto do. It compiles but the generated parser does not work for all desired combinations. Instead, have to parse optionally from and optionally to. When neither is provided, the parser succeeds, but it's a result that can't be handled. So, have to giveup after option parsing. There does not seem to be a way to make an optparse-applicative Parser give up internally either. Also, need seek' because I first tried making fto be a where binding, but that resulted in a hang when git-annex move was run without --from or --to. I think because startConcurrency was not expecting the stages value to contain an exception and so ended up blocking. Sponsored-by: Dartmouth College's DANDI project --- CHANGELOG | 2 ++ CmdLine/GitAnnex/Options.hs | 41 ++++++++++++++++++++++------------- Command/Copy.hs | 43 ++++++++++++++++++++++--------------- Command/Move.hs | 42 +++++++++++++++++++++--------------- doc/git-annex-copy.mdwn | 7 ++++++ doc/git-annex-move.mdwn | 7 ++++++ 6 files changed, 93 insertions(+), 49 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index c11d40cce7..d9293a49ff 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -23,6 +23,8 @@ git-annex (10.20221213) UNRELEASED; urgency=medium * Added an optional cost= configuration to all special remotes. * adb: Support the remote.name.cost and remote.name.cost-command configs. * findkeys: New command, very similar to git-annex find but operating on keys. + * move, copy: Support combining --from and --to in order to move/copy + the content of files that are in one remote to another remote. -- Joey Hess Mon, 12 Dec 2022 13:04:54 -0400 diff --git a/CmdLine/GitAnnex/Options.hs b/CmdLine/GitAnnex/Options.hs index 37203c2076..36434aad14 100644 --- a/CmdLine/GitAnnex/Options.hs +++ b/CmdLine/GitAnnex/Options.hs @@ -1,6 +1,6 @@ {- git-annex command-line option parsing - - - Copyright 2010-2021 Joey Hess + - Copyright 2010-2023 Joey Hess - - Licensed under the GNU AGPL version 3 or higher. -} @@ -134,7 +134,7 @@ parseDryRunOption = DryRun <$> switch <> help "don't make changes, but show what would be done" ) --- | From or To a remote. +-- | From or To a remote but not both. data FromToOptions = FromRemote (DeferredParse Remote) | ToRemote (DeferredParse Remote) @@ -162,23 +162,34 @@ parseToOption = strOption <> completeRemotes ) --- | Like FromToOptions, but with a special --to=here -type FromToHereOptions = Either ToHere FromToOptions +-- | From or to a remote, or both, or a special --to=here +data FromToHereOptions + = FromOrToRemote FromToOptions + | ToHere + | FromRemoteToRemote (DeferredParse Remote) (DeferredParse Remote) -data ToHere = ToHere - -parseFromToHereOptions :: Parser FromToHereOptions -parseFromToHereOptions = parsefrom <|> parseto +parseFromToHereOptions :: Parser (Maybe FromToHereOptions) +parseFromToHereOptions = go + <$> optional parseFromOption + <*> optional parseToOption where - parsefrom = Right . FromRemote . parseRemoteOption <$> parseFromOption - parseto = herespecialcase <$> parseToOption - where - herespecialcase "here" = Left ToHere - herespecialcase "." = Left ToHere - herespecialcase n = Right $ ToRemote $ parseRemoteOption n + go (Just from) (Just to) = Just $ FromRemoteToRemote + (parseRemoteOption from) + (parseRemoteOption to) + go (Just from) Nothing = Just $ FromOrToRemote + (FromRemote $ parseRemoteOption from) + go Nothing (Just to) = Just $ case to of + "here" -> ToHere + "." -> ToHere + _ -> FromOrToRemote $ ToRemote $ parseRemoteOption to + go Nothing Nothing = Nothing instance DeferredParseClass FromToHereOptions where - finishParse = either (pure . Left) (Right <$$> finishParse) + finishParse (FromOrToRemote v) = FromOrToRemote <$> finishParse v + finishParse ToHere = pure ToHere + finishParse (FromRemoteToRemote v1 v2) = FromRemoteToRemote + <$> finishParse v1 + <*> finishParse v2 -- Options for acting on keys, rather than work tree files. data KeyOptions diff --git a/Command/Copy.hs b/Command/Copy.hs index 267fa71d3d..11ef1ddd08 100644 --- a/Command/Copy.hs +++ b/Command/Copy.hs @@ -1,6 +1,6 @@ {- git-annex command - - - Copyright 2010 Joey Hess + - Copyright 2010-2023 Joey Hess - - Licensed under the GNU AGPL version 3 or higher. -} @@ -21,7 +21,7 @@ cmd = withAnnexOptions [jobsOption, jsonOptions, jsonProgressOption, annexedMatc data CopyOptions = CopyOptions { copyFiles :: CmdParams - , fromToOptions :: FromToHereOptions + , fromToOptions :: Maybe FromToHereOptions , keyOptions :: Maybe KeyOptions , autoMode :: Bool , batchOption :: BatchMode @@ -38,13 +38,19 @@ optParser desc = CopyOptions instance DeferredParseClass CopyOptions where finishParse v = CopyOptions <$> pure (copyFiles v) - <*> finishParse (fromToOptions v) + <*> maybe (pure Nothing) (Just <$$> finishParse) + (fromToOptions v) <*> pure (keyOptions v) <*> pure (autoMode v) <*> pure (batchOption v) seek :: CopyOptions -> CommandSeek -seek o = startConcurrency commandStages $ do +seek o = case fromToOptions o of + Just fto -> seek' o fto + Nothing -> giveup "Specify --from or --to" + +seek' :: CopyOptions -> FromToHereOptions -> CommandSeek +seek' o fto = startConcurrency commandStages $ do case batchOption o of NoBatch -> withKeyOptions (keyOptions o) (autoMode o) seeker @@ -57,30 +63,33 @@ seek o = startConcurrency commandStages $ do ww = WarnUnmatchLsFiles seeker = AnnexedFileSeeker - { startAction = start o - , checkContentPresent = case fromToOptions o of - Right (FromRemote _) -> Just False - Right (ToRemote _) -> Just True - Left ToHere -> Just False + { startAction = start o fto + , checkContentPresent = case fto of + FromOrToRemote (FromRemote _) -> Just False + FromOrToRemote (ToRemote _) -> Just True + ToHere -> Just False + FromRemoteToRemote _ _ -> Just False , usesLocationLog = True } - keyaction = Command.Move.startKey (fromToOptions o) Command.Move.RemoveNever + keyaction = Command.Move.startKey fto Command.Move.RemoveNever {- A copy is just a move that does not delete the source file. - However, auto mode avoids unnecessary copies, and avoids getting or - sending non-preferred content. -} -start :: CopyOptions -> SeekInput -> RawFilePath -> Key -> CommandStart -start o si file key = stopUnless shouldCopy $ - Command.Move.start (fromToOptions o) Command.Move.RemoveNever si file key +start :: CopyOptions -> FromToHereOptions -> SeekInput -> RawFilePath -> Key -> CommandStart +start o fto si file key = stopUnless shouldCopy $ + Command.Move.start fto Command.Move.RemoveNever si file key where shouldCopy | autoMode o = want <||> numCopiesCheck file key (<) | otherwise = return True - want = case fromToOptions o of - Right (ToRemote dest) -> + want = case fto of + FromOrToRemote (ToRemote dest) -> + (Remote.uuid <$> getParsed dest) >>= checkwantsend + FromOrToRemote (FromRemote _) -> checkwantget + ToHere -> checkwantget + FromRemoteToRemote _ dest -> (Remote.uuid <$> getParsed dest) >>= checkwantsend - Right (FromRemote _) -> checkwantget - Left ToHere -> checkwantget checkwantsend = wantGetBy False (Just key) (AssociatedFile (Just file)) checkwantget = wantGet False (Just key) (AssociatedFile (Just file)) diff --git a/Command/Move.hs b/Command/Move.hs index abe207c2f2..b14707eb27 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -1,6 +1,6 @@ {- git-annex command - - - Copyright 2010-2022 Joey Hess + - Copyright 2010-2023 Joey Hess - - Licensed under the GNU AGPL version 3 or higher. -} @@ -32,7 +32,7 @@ cmd = withAnnexOptions [jobsOption, jsonOptions, jsonProgressOption, annexedMatc data MoveOptions = MoveOptions { moveFiles :: CmdParams - , fromToOptions :: FromToHereOptions + , fromToOptions :: Maybe FromToHereOptions , removeWhen :: RemoveWhen , keyOptions :: Maybe KeyOptions , batchOption :: BatchMode @@ -49,7 +49,8 @@ optParser desc = MoveOptions instance DeferredParseClass MoveOptions where finishParse v = MoveOptions <$> pure (moveFiles v) - <*> finishParse (fromToOptions v) + <*> maybe (pure Nothing) (Just <$$> finishParse) + (fromToOptions v) <*> pure (removeWhen v) <*> pure (keyOptions v) <*> pure (batchOption v) @@ -58,7 +59,12 @@ data RemoveWhen = RemoveSafe | RemoveNever deriving (Show, Eq) seek :: MoveOptions -> CommandSeek -seek o = startConcurrency stages $ do +seek o = case fromToOptions o of + Just fto -> seek' o fto + Nothing -> giveup "Specify --from or --to" + +seek' :: MoveOptions -> FromToHereOptions -> CommandSeek +seek' o fto = startConcurrency stages $ do case batchOption o of NoBatch -> withKeyOptions (keyOptions o) False seeker (commandAction . keyaction) @@ -68,18 +74,20 @@ seek o = startConcurrency stages $ do batchAnnexed fmt seeker keyaction where seeker = AnnexedFileSeeker - { startAction = start (fromToOptions o) (removeWhen o) - , checkContentPresent = case fromToOptions o of - Right (FromRemote _) -> Nothing - Right (ToRemote _) -> Just True - Left ToHere -> Nothing + { startAction = start fto (removeWhen o) + , checkContentPresent = case fto of + FromOrToRemote (FromRemote _) -> Nothing + FromOrToRemote (ToRemote _) -> Just True + ToHere -> Nothing + FromRemoteToRemote _ _ -> Nothing , usesLocationLog = True } - stages = case fromToOptions o of - Right (FromRemote _) -> downloadStages - Right (ToRemote _) -> commandStages - Left ToHere -> downloadStages - keyaction = startKey (fromToOptions o) (removeWhen o) + stages = case fto of + FromOrToRemote (FromRemote _) -> downloadStages + FromOrToRemote (ToRemote _) -> commandStages + ToHere -> downloadStages + FromRemoteToRemote _ _ -> commandStages + keyaction = startKey fto (removeWhen o) ww = WarnUnmatchLsFiles start :: FromToHereOptions -> RemoveWhen -> SeekInput -> RawFilePath -> Key -> CommandStart @@ -95,13 +103,13 @@ startKey fromto removewhen (si, k, ai) = start' :: FromToHereOptions -> RemoveWhen -> AssociatedFile -> SeekInput -> Key -> ActionItem -> CommandStart start' fromto removewhen afile si key ai = case fromto of - Right (FromRemote src) -> + FromOrToRemote (FromRemote src) -> checkFailedTransferDirection ai Download $ fromStart removewhen afile key ai si =<< getParsed src - Right (ToRemote dest) -> + FromOrToRemote (ToRemote dest) -> checkFailedTransferDirection ai Upload $ toStart removewhen afile key ai si =<< getParsed dest - Left ToHere -> + ToHere -> checkFailedTransferDirection ai Download $ toHereStart removewhen afile key ai si diff --git a/doc/git-annex-copy.mdwn b/doc/git-annex-copy.mdwn index 5a4e805cdf..03563bda11 100644 --- a/doc/git-annex-copy.mdwn +++ b/doc/git-annex-copy.mdwn @@ -32,6 +32,13 @@ Paths of files or directories to operate on can be specified. Copy the content of files from all reachable remotes to the local repository. +* `--from=remote1 --to=remote2` + + Copy the content of files that are in remote1 to remote2. Does not change + what is stored in the local repository. + + Note: This may need to store an intermediate copy of the content on disk. + * `--jobs=N` `-JN` Enables parallel transfers with up to the specified number of jobs diff --git a/doc/git-annex-move.mdwn b/doc/git-annex-move.mdwn index 0fbea84d58..c2d5ec1e0c 100644 --- a/doc/git-annex-move.mdwn +++ b/doc/git-annex-move.mdwn @@ -28,6 +28,13 @@ Paths of files or directories to operate on can be specified. Move the content of files from all reachable remotes to the local repository. +* `--from=remote1 --to=remote2` + + Move the content of files that are in remote1 to remote2. Does not change + what is stored in the local repository. + + Note: This may need to store an intermediate copy of the content on disk. + * `--force` Override numcopies and required content checking, and always remove