copy/move --from-anywhere --to remote

Implementation was simple because it's equivilant to
--from=foo --to remote for each other remote, followed by
--to remote when there's a local copy.

(Or, in the edge case of --from-anywhere --to=here,
it's the same as --to=here.)

Note that, when the local repo does not have a copy,
fromToPerform gets it from a remote, sends it to the destination,
and drops the local copy. Another call to that for a second remote
will notice that the dest now has a copy, and simply drop from the
second remote, avoiding a second transfer.

Also note that, when numcopies doesn't allow dropping it from
everywhere, it will drop it from the cheapest remotes first
(maybe not ideal) up to more expensive remotes, and finally from the local
repo. So the local repo will generally end up holding a copy. Maybe not
ideal in all cases either, but it seems no worse to do that than to end up
with a copy undropped from a remote.

And I'm not entirely happy with the output, eg:

	copy bigfile (from r3...) ok
	copy bigfile ok

That makes sense if you think of the second line as being
the same as what is output by `git-annex copy bigfile --to bar`,
but it's less clear in this context. Maybe add "(from here...)"?
Also the --json output doesn't have a machine-readable field for
the "from" uuid, and maybe it should?

Sponsored-by: Dartmouth College's DANDI project

CHANGELOG

@@ -2,6 +2,7 @@ git-annex (10.20231130) UNRELEASED; urgency=medium
 
   * Make git-annex get/copy/move --from foo override configuration of 
     remote.foo.annex-ignore, as documented.
+  * Support git-annex copy/move --from-anywhere --to remote.
 
  -- Joey Hess <id@joeyh.name>  Thu, 30 Nov 2023 14:48:12 -0400
 

CmdLine/GitAnnex/Options.hs

@@ -162,40 +162,55 @@ parseToOption = strOption
 	<> completeRemotes
 	)
 
+parseFromAnywhereOption :: Parser Bool
+parseFromAnywhereOption = switch
+	( long "from-anywhere"
+	<> help "from any remote"
+	)
+
 parseRemoteOption :: Parser RemoteName
 parseRemoteOption = strOption
 	( long "remote" <> metavar paramRemote
 	<> completeRemotes
 	)
 
--- | From or to a remote, or both, or a special --to=here
+-- | --from or --to a remote, or both, or a special --to=here,
+-- or --from-anywhere --to remote.
 data FromToHereOptions 
 	= FromOrToRemote FromToOptions
 	| ToHere
 	| FromRemoteToRemote (DeferredParse Remote) (DeferredParse Remote)
+	| FromAnywhereToRemote (DeferredParse Remote)
 
 parseFromToHereOptions :: Parser (Maybe FromToHereOptions)
 parseFromToHereOptions = go
 	<$> optional parseFromOption
 	<*> optional parseToOption
+	<*> parseFromAnywhereOption
   where
-	go (Just from) (Just to) = Just $ FromRemoteToRemote
+	go _ (Just to) True = Just $ FromAnywhereToRemote
+		(mkParseRemoteOption to)
+	go (Just from) (Just to) _ = Just $ FromRemoteToRemote
 		(mkParseRemoteOption from)
 		(mkParseRemoteOption to)
-	go (Just from) Nothing = Just $ FromOrToRemote
+	go (Just from) Nothing _ = Just $ FromOrToRemote
 		(FromRemote $ mkParseRemoteOption from)
-	go Nothing (Just to) = Just $ case to of
+	go Nothing (Just to) _ = Just $ case to of
 		"here" -> ToHere
 		"." -> ToHere
 		_ -> FromOrToRemote $ ToRemote $ mkParseRemoteOption to
-	go Nothing Nothing = Nothing
+	go Nothing Nothing _ = Nothing
 
 instance DeferredParseClass FromToHereOptions where
-	finishParse (FromOrToRemote v) = FromOrToRemote <$> finishParse v
+	finishParse (FromOrToRemote v) = 
+		FromOrToRemote <$> finishParse v
 	finishParse ToHere = pure ToHere
-	finishParse (FromRemoteToRemote v1 v2) = FromRemoteToRemote
+	finishParse (FromRemoteToRemote v1 v2) = 
+		FromRemoteToRemote
 			<$> finishParse v1
 			<*> finishParse v2
+	finishParse (FromAnywhereToRemote v) = 
+		FromAnywhereToRemote <$> finishParse v
 
 -- Options for acting on keys, rather than work tree files.
 data KeyOptions

Command/Copy.hs

@@ -69,6 +69,7 @@ seek' o fto = startConcurrency (Command.Move.stages fto) $ do
 			FromOrToRemote (ToRemote _) -> Just True
 			ToHere -> Just False
 			FromRemoteToRemote _ _ -> Nothing
+			FromAnywhereToRemote _ -> Nothing
 		, usesLocationLog = True
 		}
 	keyaction = Command.Move.startKey fto Command.Move.RemoveNever
@@ -84,12 +85,13 @@ start o fto si file key = stopUnless shouldCopy $
 		| autoMode o = want <||> numCopiesCheck file key (<)
 		| otherwise = return True
 	want = case fto of
-		FromOrToRemote (ToRemote dest) ->
-			(Remote.uuid <$> getParsed dest) >>= checkwantsend
+		FromOrToRemote (ToRemote dest) -> checkwantsend dest
 		FromOrToRemote (FromRemote _) -> checkwantget
 		ToHere -> checkwantget
-		FromRemoteToRemote _ dest ->
-			(Remote.uuid <$> getParsed dest) >>= checkwantsend
+		FromRemoteToRemote _ dest -> checkwantsend dest
+		FromAnywhereToRemote dest -> checkwantsend dest
 
-	checkwantsend = wantGetBy False (Just key) (AssociatedFile (Just file))
+	checkwantsend dest = 
+		(Remote.uuid <$> getParsed dest) >>=
+			wantGetBy False (Just key) (AssociatedFile (Just file))
 	checkwantget = wantGet False (Just key) (AssociatedFile (Just file))

Command/Move.hs

@@ -81,6 +81,7 @@ seek' o fto = startConcurrency (stages fto) $ do
 			FromOrToRemote (ToRemote _) -> Just True
 			ToHere -> Nothing
 			FromRemoteToRemote _ _ -> Nothing
+			FromAnywhereToRemote _ -> Nothing
 		, usesLocationLog = True
 		}
 	keyaction = startKey fto (removeWhen o)
@@ -91,6 +92,7 @@ stages (FromOrToRemote (FromRemote _)) = transferStages
 stages (FromOrToRemote (ToRemote _)) = commandStages
 stages ToHere = transferStages
 stages (FromRemoteToRemote _ _) = transferStages
+stages (FromAnywhereToRemote _) = transferStages
 
 start :: FromToHereOptions -> RemoveWhen -> SeekInput -> RawFilePath -> Key -> CommandStart
 start fromto removewhen si f k = start' fromto removewhen afile si k ai
@@ -118,6 +120,9 @@ start' fromto removewhen afile si key ai =
 			src' <- getParsed src
 			dest' <- getParsed dest
 			fromToStart removewhen afile key ai si src' dest'
+		FromAnywhereToRemote dest -> do
+			dest' <- getParsed dest
+			fromAnywhereToStart removewhen afile key ai si dest'
 
 describeMoveAction :: RemoveWhen -> String
 describeMoveAction RemoveNever = "copy"
@@ -353,6 +358,30 @@ fromToStart removewhen afile key ai si src dest =
 				then not <$> expectedPresent dest key
 				else return True
 
+fromAnywhereToStart :: RemoveWhen -> AssociatedFile -> Key -> ActionItem -> SeekInput -> Remote -> CommandStart
+fromAnywhereToStart removewhen afile key ai si dest =
+	stopUnless somethingtodo $ do
+		u <- getUUID
+		if u == Remote.uuid dest
+			then toHereStart removewhen afile key ai si
+			else startingNoMessage (OnlyActionOn key ai) $ do
+				rs <- filter (/= dest) 
+					<$> Remote.keyPossibilities (Remote.IncludeIgnored False) key
+				forM_ rs $ \r ->
+					includeCommandAction $
+						starting (describeMoveAction removewhen) ai si $
+							fromToPerform r dest removewhen key afile
+				whenM (inAnnex key) $
+					void $ includeCommandAction $
+						toStart removewhen afile key ai si dest 
+				next $ return True
+  where
+	somethingtodo = do
+		fast <- Annex.getRead Annex.fast
+		if fast && removewhen == RemoveNever
+			then not <$> expectedPresent dest key
+			else return True
+
 {- When there is a local copy, transfer it to the dest, and drop from the src.
  -
  - When the dest has a copy, drop it from the src.

doc/bugs/copy_--from_--to_does_not_copy_if_present_locally/comment_6_c374eb44ea08f220dbcce5ecb88403fb._comment

@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2023-11-30T18:26:30Z"
+ content="""
+I like the idea of `copy --from-anywhere --to=remote` and just 
+use the lowest cost remote (when not in local repo). Like `git-annex get`
+and `git-annex copy --to=here`.
+
+Hmm, if there's a remote that is too expensive to want to use in such a
+copy, it would be possible to use `-c remote.foo.annex-ignore=true`
+to make it avoid using that remote. As can also be done in the case of
+`git-annex get`, although that was not documented well.
+
+I've implemented --from-anywhere..
+"""]]

doc/git-annex-copy.mdwn

@@ -41,6 +41,11 @@ Paths of files or directories to operate on can be specified.
   then deleting the content from the local repository (if it was not present
   to start with).
 
+* `--from-anywhere --to=remote`
+
+  Copy to the remote files from the local repository as well as from any reachable
+  remotes.
+
 * `--jobs=N` `-JN`
 
   Enables parallel transfers with up to the specified number of jobs

doc/git-annex-move.mdwn

@@ -38,6 +38,11 @@ Paths of files or directories to operate on can be specified.
   then deleting the content from the local repository (if it was not present
   to start with).
 
+* `--from-anywhere --to=remote`
+
+  Move to the remote files from the local repository and from all 
+  reachable remotes.
+
 * `--force`
 
   Override numcopies and required content checking, and always remove