move/copy: option parsing for --from with --to

Allowing --from and --to as an alternative to --from or --to is hard to do with optparse-applicative! The obvious approach of (pfrom <|> pto <|> pfromandto) does not work when pfromandto uses the same option names as pfrom and pto do. It compiles but the generated parser does not work for all desired combinations. Instead, have to parse optionally from and optionally to. When neither is provided, the parser succeeds, but it's a result that can't be handled. So, have to giveup after option parsing. There does not seem to be a way to make an optparse-applicative Parser give up internally either. Also, need seek' because I first tried making fto be a where binding, but that resulted in a hang when git-annex move was run without --from or --to. I think because startConcurrency was not expecting the stages value to contain an exception and so ended up blocking. Sponsored-by: Dartmouth College's DANDI project
2023-01-18 14:42:39 -04:00 · 2023-01-18 14:42:39 -04:00 · a6c1d9752b
commit a6c1d9752b
parent 2a92f5cc2c
6 changed files with 93 additions and 49 deletions
--- a/2
+++ b/2
@ -23,6 +23,8 @@ git-annex (10.20221213) UNRELEASED; urgency=medium
  * Added an optional cost= configuration to all special remotes.
  * adb: Support the remote.name.cost and remote.name.cost-command configs.
  * findkeys: New command, very similar to git-annex find but operating on keys.
+  * move, copy: Support combining --from and --to in order to move/copy
+    the content of files that are in one remote to another remote.

 -- Joey Hess <id@joeyh.name>  Mon, 12 Dec 2022 13:04:54 -0400

--- a/CmdLine/GitAnnex/Options.hs
+++ b/CmdLine/GitAnnex/Options.hs
@ -1,6 +1,6 @@
 {- git-annex command-line option parsing
 -
- - Copyright 2010-2021 Joey Hess <id@joeyh.name>
+ - Copyright 2010-2023 Joey Hess <id@joeyh.name>
 -
 - Licensed under the GNU AGPL version 3 or higher.
 -}
@ -134,7 +134,7 @@ parseDryRunOption = DryRun <$> switch
 	<> help "don't make changes, but show what would be done"
 	)

-- | From or To a remote.
+-- | From or To a remote but not both.
 data FromToOptions
 	= FromRemote (DeferredParse Remote)
 	| ToRemote (DeferredParse Remote)
@ -162,23 +162,34 @@ parseToOption = strOption
 	<> completeRemotes
 	)

-- | Like FromToOptions, but with a special --to=here
-type FromToHereOptions = Either ToHere FromToOptions
+-- | From or to a remote, or both, or a special --to=here
+data FromToHereOptions 
+	= FromOrToRemote FromToOptions
+	| ToHere
+	| FromRemoteToRemote (DeferredParse Remote) (DeferredParse Remote)

-data ToHere = ToHere
-
-parseFromToHereOptions :: Parser FromToHereOptions
-parseFromToHereOptions = parsefrom <|> parseto
+parseFromToHereOptions :: Parser (Maybe FromToHereOptions)
+parseFromToHereOptions = go
+	<$> optional parseFromOption
+	<*> optional parseToOption
  where
-	parsefrom = Right . FromRemote . parseRemoteOption <$> parseFromOption
-	parseto = herespecialcase <$> parseToOption
-	  where
-		herespecialcase "here" = Left ToHere
-		herespecialcase "." = Left ToHere
-		herespecialcase n = Right $ ToRemote $ parseRemoteOption n
+	go (Just from) (Just to) = Just $ FromRemoteToRemote
+		(parseRemoteOption from)
+		(parseRemoteOption to)
+	go (Just from) Nothing = Just $ FromOrToRemote
+		(FromRemote $ parseRemoteOption from)
+	go Nothing (Just to) = Just $ case to of
+		"here" -> ToHere
+		"." -> ToHere
+		_ -> FromOrToRemote $ ToRemote $ parseRemoteOption to
+	go Nothing Nothing = Nothing

 instance DeferredParseClass FromToHereOptions where
-	finishParse = either (pure . Left) (Right <$$> finishParse)
+	finishParse (FromOrToRemote v) = FromOrToRemote <$> finishParse v
+	finishParse ToHere = pure ToHere
+	finishParse (FromRemoteToRemote v1 v2) = FromRemoteToRemote
+		<$> finishParse v1
+		<*> finishParse v2

 -- Options for acting on keys, rather than work tree files.
 data KeyOptions
--- a/Command/Copy.hs
+++ b/Command/Copy.hs
@ -1,6 +1,6 @@
 {- git-annex command
 -
- - Copyright 2010 Joey Hess <id@joeyh.name>
+ - Copyright 2010-2023 Joey Hess <id@joeyh.name>
 -
 - Licensed under the GNU AGPL version 3 or higher.
 -}
@ -21,7 +21,7 @@ cmd = withAnnexOptions [jobsOption, jsonOptions, jsonProgressOption, annexedMatc

 data CopyOptions = CopyOptions
 	{ copyFiles :: CmdParams
-	, fromToOptions :: FromToHereOptions
+	, fromToOptions :: Maybe FromToHereOptions
 	, keyOptions :: Maybe KeyOptions
 	, autoMode :: Bool
 	, batchOption :: BatchMode
@ -38,13 +38,19 @@ optParser desc = CopyOptions
 instance DeferredParseClass CopyOptions where
 	finishParse v = CopyOptions
 		<$> pure (copyFiles v)
-		<*> finishParse (fromToOptions v)
+		<*> maybe (pure Nothing) (Just <$$> finishParse)
+			(fromToOptions v)
 		<*> pure (keyOptions v)
 		<*> pure (autoMode v)
 		<*> pure (batchOption v)

 seek :: CopyOptions -> CommandSeek
-seek o = startConcurrency commandStages $ do
+seek o = case fromToOptions o of
+	Just fto -> seek' o fto
+	Nothing -> giveup "Specify --from or --to"
+
+seek' :: CopyOptions -> FromToHereOptions -> CommandSeek
+seek' o fto = startConcurrency commandStages $ do
 	case batchOption o of
 		NoBatch -> withKeyOptions
 			(keyOptions o) (autoMode o) seeker
@ -57,30 +63,33 @@ seek o = startConcurrency commandStages $ do
 	ww = WarnUnmatchLsFiles
 	
 	seeker = AnnexedFileSeeker
-		{ startAction = start o
-		, checkContentPresent = case fromToOptions o of
-			Right (FromRemote _) -> Just False
-			Right (ToRemote _) -> Just True
-			Left ToHere -> Just False
+		{ startAction = start o fto
+		, checkContentPresent = case fto of
+			FromOrToRemote (FromRemote _) -> Just False
+			FromOrToRemote (ToRemote _) -> Just True
+			ToHere -> Just False
+			FromRemoteToRemote _ _ -> Just False
 		, usesLocationLog = True
 		}
-	keyaction = Command.Move.startKey (fromToOptions o) Command.Move.RemoveNever
+	keyaction = Command.Move.startKey fto Command.Move.RemoveNever

 {- A copy is just a move that does not delete the source file.
 - However, auto mode avoids unnecessary copies, and avoids getting or
 - sending non-preferred content. -}
-start :: CopyOptions -> SeekInput -> RawFilePath -> Key -> CommandStart
-start o si file key = stopUnless shouldCopy $ 
-	Command.Move.start (fromToOptions o) Command.Move.RemoveNever si file key
+start :: CopyOptions -> FromToHereOptions -> SeekInput -> RawFilePath -> Key -> CommandStart
+start o fto si file key = stopUnless shouldCopy $ 
+	Command.Move.start fto Command.Move.RemoveNever si file key
  where
 	shouldCopy
 		| autoMode o = want <||> numCopiesCheck file key (<)
 		| otherwise = return True
-	want = case fromToOptions o of
-		Right (ToRemote dest) ->
+	want = case fto of
+		FromOrToRemote (ToRemote dest) ->
+			(Remote.uuid <$> getParsed dest) >>= checkwantsend
+		FromOrToRemote (FromRemote _) -> checkwantget
+		ToHere -> checkwantget
+		FromRemoteToRemote _ dest ->
 			(Remote.uuid <$> getParsed dest) >>= checkwantsend
-		Right (FromRemote _) -> checkwantget
-		Left ToHere -> checkwantget
 			
 	checkwantsend = wantGetBy False (Just key) (AssociatedFile (Just file))
 	checkwantget = wantGet False (Just key) (AssociatedFile (Just file))
--- a/Command/Move.hs
+++ b/Command/Move.hs
@ -1,6 +1,6 @@
 {- git-annex command
 -
- - Copyright 2010-2022 Joey Hess <id@joeyh.name>
+ - Copyright 2010-2023 Joey Hess <id@joeyh.name>
 -
 - Licensed under the GNU AGPL version 3 or higher.
 -}
@ -32,7 +32,7 @@ cmd = withAnnexOptions [jobsOption, jsonOptions, jsonProgressOption, annexedMatc

 data MoveOptions = MoveOptions
 	{ moveFiles :: CmdParams
-	, fromToOptions :: FromToHereOptions
+	, fromToOptions :: Maybe FromToHereOptions
 	, removeWhen :: RemoveWhen
 	, keyOptions :: Maybe KeyOptions
 	, batchOption :: BatchMode
@ -49,7 +49,8 @@ optParser desc = MoveOptions
 instance DeferredParseClass MoveOptions where
 	finishParse v = MoveOptions
 		<$> pure (moveFiles v)
-		<*> finishParse (fromToOptions v)
+		<*> maybe (pure Nothing) (Just <$$> finishParse)
+			(fromToOptions v)
 		<*> pure (removeWhen v)
 		<*> pure (keyOptions v)
 		<*> pure (batchOption v)
@ -58,7 +59,12 @@ data RemoveWhen = RemoveSafe | RemoveNever
 	deriving (Show, Eq)

 seek :: MoveOptions -> CommandSeek
-seek o = startConcurrency stages $ do
+seek o = case fromToOptions o of
+	Just fto -> seek' o fto
+	Nothing -> giveup "Specify --from or --to"
+
+seek' :: MoveOptions -> FromToHereOptions -> CommandSeek
+seek' o fto = startConcurrency stages $ do
 	case batchOption o of
 		NoBatch -> withKeyOptions (keyOptions o) False seeker
 			(commandAction . keyaction)
@ -68,18 +74,20 @@ seek o = startConcurrency stages $ do
 			batchAnnexed fmt seeker keyaction
  where
 	seeker = AnnexedFileSeeker
-		{ startAction = start (fromToOptions o) (removeWhen o)
-		, checkContentPresent = case fromToOptions o of
-			Right (FromRemote _) -> Nothing
-			Right (ToRemote _) -> Just True
-			Left ToHere -> Nothing
+		{ startAction = start fto (removeWhen o)
+		, checkContentPresent = case fto of
+			FromOrToRemote (FromRemote _) -> Nothing
+			FromOrToRemote (ToRemote _) -> Just True
+			ToHere -> Nothing
+			FromRemoteToRemote _ _ -> Nothing
 		, usesLocationLog = True
 		}
-	stages = case fromToOptions o of
-		Right (FromRemote _) -> downloadStages
-		Right (ToRemote _) -> commandStages
-		Left ToHere -> downloadStages
-	keyaction = startKey (fromToOptions o) (removeWhen o)
+	stages = case fto of
+		FromOrToRemote (FromRemote _) -> downloadStages
+		FromOrToRemote (ToRemote _) -> commandStages
+		ToHere -> downloadStages
+		FromRemoteToRemote _ _ -> commandStages
+	keyaction = startKey fto (removeWhen o)
 	ww = WarnUnmatchLsFiles

 start :: FromToHereOptions -> RemoveWhen -> SeekInput -> RawFilePath -> Key -> CommandStart
@ -95,13 +103,13 @@ startKey fromto removewhen (si, k, ai) =
 start' :: FromToHereOptions -> RemoveWhen -> AssociatedFile -> SeekInput -> Key -> ActionItem -> CommandStart
 start' fromto removewhen afile si key ai =
 	case fromto of
-		Right (FromRemote src) ->
+		FromOrToRemote (FromRemote src) ->
 			checkFailedTransferDirection ai Download $
 				fromStart removewhen afile key ai si =<< getParsed src
-		Right (ToRemote dest) ->
+		FromOrToRemote (ToRemote dest) ->
 			checkFailedTransferDirection ai Upload $
 				toStart removewhen afile key ai si =<< getParsed dest
-		Left ToHere ->
+		ToHere ->
 			checkFailedTransferDirection ai Download $
 				toHereStart removewhen afile key ai si

--- a/doc/git-annex-copy.mdwn
+++ b/doc/git-annex-copy.mdwn
@ -32,6 +32,13 @@ Paths of files or directories to operate on can be specified.
  Copy the content of files from all reachable remotes to the local
  repository.

+* `--from=remote1 --to=remote2`
+
+  Copy the content of files that are in remote1 to remote2. Does not change
+  what is stored in the local repository.
+
+  Note: This may need to store an intermediate copy of the content on disk.
+
 * `--jobs=N` `-JN`

  Enables parallel transfers with up to the specified number of jobs
--- a/doc/git-annex-move.mdwn
+++ b/doc/git-annex-move.mdwn
@ -28,6 +28,13 @@ Paths of files or directories to operate on can be specified.
  Move the content of files from all reachable remotes to the local
  repository.

+* `--from=remote1 --to=remote2`
+
+  Move the content of files that are in remote1 to remote2. Does not change
+  what is stored in the local repository. 
+
+  Note: This may need to store an intermediate copy of the content on disk.
+
 * `--force`

  Override numcopies and required content checking, and always remove