move/copy: option parsing for --from with --to

Allowing --from and --to as an alternative to --from or --to
is hard to do with optparse-applicative!

The obvious approach of (pfrom <|> pto <|> pfromandto) does not work
when pfromandto uses the same option names as pfrom and pto do.
It compiles but the generated parser does not work for all desired
combinations.

Instead, have to parse optionally from and optionally to. When neither
is provided, the parser succeeds, but it's a result that can't be
handled. So, have to giveup after option parsing. There does not seem to
be a way to make an optparse-applicative Parser give up internally
either.

Also, need seek' because I first tried making fto be a where binding,
but that resulted in a hang when git-annex move was run without --from
or --to. I think because startConcurrency was not expecting the stages
value to contain an exception and so ended up blocking.

Sponsored-by: Dartmouth College's DANDI project

CHANGELOG

@@ -23,6 +23,8 @@ git-annex (10.20221213) UNRELEASED; urgency=medium
   * Added an optional cost= configuration to all special remotes.
   * adb: Support the remote.name.cost and remote.name.cost-command configs.
   * findkeys: New command, very similar to git-annex find but operating on keys.
+  * move, copy: Support combining --from and --to in order to move/copy
+    the content of files that are in one remote to another remote.
 
  -- Joey Hess <id@joeyh.name>  Mon, 12 Dec 2022 13:04:54 -0400
 

CmdLine/GitAnnex/Options.hs

@@ -1,6 +1,6 @@
 {- git-annex command-line option parsing
  -
- - Copyright 2010-2021 Joey Hess <id@joeyh.name>
+ - Copyright 2010-2023 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
@@ -134,7 +134,7 @@ parseDryRunOption = DryRun <$> switch
 	<> help "don't make changes, but show what would be done"
 	)
 
--- | From or To a remote.
+-- | From or To a remote but not both.
 data FromToOptions
 	= FromRemote (DeferredParse Remote)
 	| ToRemote (DeferredParse Remote)
@@ -162,23 +162,34 @@ parseToOption = strOption
 	<> completeRemotes
 	)
 
--- | Like FromToOptions, but with a special --to=here
+-- | From or to a remote, or both, or a special --to=here
-type FromToHereOptions = Either ToHere FromToOptions
+data FromToHereOptions 
+	= FromOrToRemote FromToOptions
+	| ToHere
+	| FromRemoteToRemote (DeferredParse Remote) (DeferredParse Remote)
 
-data ToHere = ToHere
+parseFromToHereOptions :: Parser (Maybe FromToHereOptions)
-
+parseFromToHereOptions = go
-parseFromToHereOptions :: Parser FromToHereOptions
+	<$> optional parseFromOption
-parseFromToHereOptions = parsefrom <|> parseto
+	<*> optional parseToOption
   where
-	parsefrom = Right . FromRemote . parseRemoteOption <$> parseFromOption
+	go (Just from) (Just to) = Just $ FromRemoteToRemote
-	parseto = herespecialcase <$> parseToOption
+		(parseRemoteOption from)
-	  where
+		(parseRemoteOption to)
-		herespecialcase "here" = Left ToHere
+	go (Just from) Nothing = Just $ FromOrToRemote
-		herespecialcase "." = Left ToHere
+		(FromRemote $ parseRemoteOption from)
-		herespecialcase n = Right $ ToRemote $ parseRemoteOption n
+	go Nothing (Just to) = Just $ case to of
+		"here" -> ToHere
+		"." -> ToHere
+		_ -> FromOrToRemote $ ToRemote $ parseRemoteOption to
+	go Nothing Nothing = Nothing
 
 instance DeferredParseClass FromToHereOptions where
-	finishParse = either (pure . Left) (Right <$$> finishParse)
+	finishParse (FromOrToRemote v) = FromOrToRemote <$> finishParse v
+	finishParse ToHere = pure ToHere
+	finishParse (FromRemoteToRemote v1 v2) = FromRemoteToRemote
+		<$> finishParse v1
+		<*> finishParse v2
 
 -- Options for acting on keys, rather than work tree files.
 data KeyOptions

Command/Copy.hs

@@ -1,6 +1,6 @@
 {- git-annex command
  -
- - Copyright 2010 Joey Hess <id@joeyh.name>
+ - Copyright 2010-2023 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
@@ -21,7 +21,7 @@ cmd = withAnnexOptions [jobsOption, jsonOptions, jsonProgressOption, annexedMatc
 
 data CopyOptions = CopyOptions
 	{ copyFiles :: CmdParams
-	, fromToOptions :: FromToHereOptions
+	, fromToOptions :: Maybe FromToHereOptions
 	, keyOptions :: Maybe KeyOptions
 	, autoMode :: Bool
 	, batchOption :: BatchMode
@@ -38,13 +38,19 @@ optParser desc = CopyOptions
 instance DeferredParseClass CopyOptions where
 	finishParse v = CopyOptions
 		<$> pure (copyFiles v)
-		<*> finishParse (fromToOptions v)
+		<*> maybe (pure Nothing) (Just <$$> finishParse)
+			(fromToOptions v)
 		<*> pure (keyOptions v)
 		<*> pure (autoMode v)
 		<*> pure (batchOption v)
 
 seek :: CopyOptions -> CommandSeek
-seek o = startConcurrency commandStages $ do
+seek o = case fromToOptions o of
+	Just fto -> seek' o fto
+	Nothing -> giveup "Specify --from or --to"
+
+seek' :: CopyOptions -> FromToHereOptions -> CommandSeek
+seek' o fto = startConcurrency commandStages $ do
 	case batchOption o of
 		NoBatch -> withKeyOptions
 			(keyOptions o) (autoMode o) seeker
@@ -57,30 +63,33 @@ seek o = startConcurrency commandStages $ do
 	ww = WarnUnmatchLsFiles
 	
 	seeker = AnnexedFileSeeker
-		{ startAction = start o
+		{ startAction = start o fto
-		, checkContentPresent = case fromToOptions o of
+		, checkContentPresent = case fto of
-			Right (FromRemote _) -> Just False
+			FromOrToRemote (FromRemote _) -> Just False
-			Right (ToRemote _) -> Just True
+			FromOrToRemote (ToRemote _) -> Just True
-			Left ToHere -> Just False
+			ToHere -> Just False
+			FromRemoteToRemote _ _ -> Just False
 		, usesLocationLog = True
 		}
-	keyaction = Command.Move.startKey (fromToOptions o) Command.Move.RemoveNever
+	keyaction = Command.Move.startKey fto Command.Move.RemoveNever
 
 {- A copy is just a move that does not delete the source file.
  - However, auto mode avoids unnecessary copies, and avoids getting or
  - sending non-preferred content. -}
-start :: CopyOptions -> SeekInput -> RawFilePath -> Key -> CommandStart
+start :: CopyOptions -> FromToHereOptions -> SeekInput -> RawFilePath -> Key -> CommandStart
-start o si file key = stopUnless shouldCopy $ 
+start o fto si file key = stopUnless shouldCopy $ 
-	Command.Move.start (fromToOptions o) Command.Move.RemoveNever si file key
+	Command.Move.start fto Command.Move.RemoveNever si file key
   where
 	shouldCopy
 		| autoMode o = want <||> numCopiesCheck file key (<)
 		| otherwise = return True
-	want = case fromToOptions o of
+	want = case fto of
-		Right (ToRemote dest) ->
+		FromOrToRemote (ToRemote dest) ->
+			(Remote.uuid <$> getParsed dest) >>= checkwantsend
+		FromOrToRemote (FromRemote _) -> checkwantget
+		ToHere -> checkwantget
+		FromRemoteToRemote _ dest ->
 			(Remote.uuid <$> getParsed dest) >>= checkwantsend
-		Right (FromRemote _) -> checkwantget
-		Left ToHere -> checkwantget
 			
 	checkwantsend = wantGetBy False (Just key) (AssociatedFile (Just file))
 	checkwantget = wantGet False (Just key) (AssociatedFile (Just file))

Command/Move.hs

@@ -1,6 +1,6 @@
 {- git-annex command
  -
- - Copyright 2010-2022 Joey Hess <id@joeyh.name>
+ - Copyright 2010-2023 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
@@ -32,7 +32,7 @@ cmd = withAnnexOptions [jobsOption, jsonOptions, jsonProgressOption, annexedMatc
 
 data MoveOptions = MoveOptions
 	{ moveFiles :: CmdParams
-	, fromToOptions :: FromToHereOptions
+	, fromToOptions :: Maybe FromToHereOptions
 	, removeWhen :: RemoveWhen
 	, keyOptions :: Maybe KeyOptions
 	, batchOption :: BatchMode
@@ -49,7 +49,8 @@ optParser desc = MoveOptions
 instance DeferredParseClass MoveOptions where
 	finishParse v = MoveOptions
 		<$> pure (moveFiles v)
-		<*> finishParse (fromToOptions v)
+		<*> maybe (pure Nothing) (Just <$$> finishParse)
+			(fromToOptions v)
 		<*> pure (removeWhen v)
 		<*> pure (keyOptions v)
 		<*> pure (batchOption v)
@@ -58,7 +59,12 @@ data RemoveWhen = RemoveSafe | RemoveNever
 	deriving (Show, Eq)
 
 seek :: MoveOptions -> CommandSeek
-seek o = startConcurrency stages $ do
+seek o = case fromToOptions o of
+	Just fto -> seek' o fto
+	Nothing -> giveup "Specify --from or --to"
+
+seek' :: MoveOptions -> FromToHereOptions -> CommandSeek
+seek' o fto = startConcurrency stages $ do
 	case batchOption o of
 		NoBatch -> withKeyOptions (keyOptions o) False seeker
 			(commandAction . keyaction)
@@ -68,18 +74,20 @@ seek o = startConcurrency stages $ do
 			batchAnnexed fmt seeker keyaction
   where
 	seeker = AnnexedFileSeeker
-		{ startAction = start (fromToOptions o) (removeWhen o)
+		{ startAction = start fto (removeWhen o)
-		, checkContentPresent = case fromToOptions o of
+		, checkContentPresent = case fto of
-			Right (FromRemote _) -> Nothing
+			FromOrToRemote (FromRemote _) -> Nothing
-			Right (ToRemote _) -> Just True
+			FromOrToRemote (ToRemote _) -> Just True
-			Left ToHere -> Nothing
+			ToHere -> Nothing
+			FromRemoteToRemote _ _ -> Nothing
 		, usesLocationLog = True
 		}
-	stages = case fromToOptions o of
+	stages = case fto of
-		Right (FromRemote _) -> downloadStages
+		FromOrToRemote (FromRemote _) -> downloadStages
-		Right (ToRemote _) -> commandStages
+		FromOrToRemote (ToRemote _) -> commandStages
-		Left ToHere -> downloadStages
+		ToHere -> downloadStages
-	keyaction = startKey (fromToOptions o) (removeWhen o)
+		FromRemoteToRemote _ _ -> commandStages
+	keyaction = startKey fto (removeWhen o)
 	ww = WarnUnmatchLsFiles
 
 start :: FromToHereOptions -> RemoveWhen -> SeekInput -> RawFilePath -> Key -> CommandStart
@@ -95,13 +103,13 @@ startKey fromto removewhen (si, k, ai) =
 start' :: FromToHereOptions -> RemoveWhen -> AssociatedFile -> SeekInput -> Key -> ActionItem -> CommandStart
 start' fromto removewhen afile si key ai =
 	case fromto of
-		Right (FromRemote src) ->
+		FromOrToRemote (FromRemote src) ->
 			checkFailedTransferDirection ai Download $
 				fromStart removewhen afile key ai si =<< getParsed src
-		Right (ToRemote dest) ->
+		FromOrToRemote (ToRemote dest) ->
 			checkFailedTransferDirection ai Upload $
 				toStart removewhen afile key ai si =<< getParsed dest
-		Left ToHere ->
+		ToHere ->
 			checkFailedTransferDirection ai Download $
 				toHereStart removewhen afile key ai si
 

doc/git-annex-copy.mdwn

@@ -32,6 +32,13 @@ Paths of files or directories to operate on can be specified.
   Copy the content of files from all reachable remotes to the local
   repository.
 
+* `--from=remote1 --to=remote2`
+
+  Copy the content of files that are in remote1 to remote2. Does not change
+  what is stored in the local repository.
+
+  Note: This may need to store an intermediate copy of the content on disk.
+
 * `--jobs=N` `-JN`
 
   Enables parallel transfers with up to the specified number of jobs

doc/git-annex-move.mdwn

@@ -28,6 +28,13 @@ Paths of files or directories to operate on can be specified.
   Move the content of files from all reachable remotes to the local
   repository.
 
+* `--from=remote1 --to=remote2`
+
+  Move the content of files that are in remote1 to remote2. Does not change
+  what is stored in the local repository. 
+
+  Note: This may need to store an intermediate copy of the content on disk.
+
 * `--force`
 
   Override numcopies and required content checking, and always remove