2010-11-02 23:04:24 +00:00
|
|
|
{- git-annex command
|
|
|
|
-
|
2024-06-23 16:00:50 +00:00
|
|
|
- Copyright 2010-2024 Joey Hess <id@joeyh.name>
|
2010-11-02 23:04:24 +00:00
|
|
|
-
|
2019-03-13 19:48:14 +00:00
|
|
|
- Licensed under the GNU AGPL version 3 or higher.
|
2010-11-02 23:04:24 +00:00
|
|
|
-}
|
|
|
|
|
2021-04-05 17:40:31 +00:00
|
|
|
{-# LANGUAGE OverloadedStrings #-}
|
|
|
|
|
2010-11-02 23:04:24 +00:00
|
|
|
module Command.Move where
|
|
|
|
|
|
|
|
import Command
|
2010-11-11 22:54:52 +00:00
|
|
|
import qualified Command.Drop
|
2010-11-02 23:04:24 +00:00
|
|
|
import qualified Annex
|
2011-10-04 04:40:47 +00:00
|
|
|
import Annex.Content
|
2011-03-27 21:24:20 +00:00
|
|
|
import qualified Remote
|
2011-10-15 21:47:03 +00:00
|
|
|
import Annex.UUID
|
2014-03-22 14:42:38 +00:00
|
|
|
import Annex.Transfer
|
2018-04-13 18:06:25 +00:00
|
|
|
import Logs.Trust
|
2020-10-21 14:31:56 +00:00
|
|
|
import Logs.File
|
2023-01-23 17:45:26 +00:00
|
|
|
import Logs.Location
|
2015-10-09 20:16:03 +00:00
|
|
|
import Annex.NumCopies
|
2024-06-23 16:00:50 +00:00
|
|
|
import Types.Cluster
|
2015-10-09 20:16:03 +00:00
|
|
|
|
2020-10-21 14:31:56 +00:00
|
|
|
import qualified Data.ByteString.Char8 as B8
|
|
|
|
import qualified Data.ByteString.Lazy as L
|
2011-03-16 01:34:13 +00:00
|
|
|
|
2015-07-08 16:33:27 +00:00
|
|
|
cmd :: Command
|
2022-06-29 17:28:08 +00:00
|
|
|
cmd = withAnnexOptions [jobsOption, jsonOptions, jsonProgressOption, annexedMatchingOptions] $
|
2015-07-10 17:18:46 +00:00
|
|
|
command "move" SectionCommon
|
|
|
|
"move content of files to/from another repository"
|
|
|
|
paramPaths (seek <--< optParser)
|
2015-07-09 19:23:14 +00:00
|
|
|
|
|
|
|
data MoveOptions = MoveOptions
|
|
|
|
{ moveFiles :: CmdParams
|
2023-01-18 18:42:39 +00:00
|
|
|
, fromToOptions :: Maybe FromToHereOptions
|
2018-04-13 18:06:25 +00:00
|
|
|
, removeWhen :: RemoveWhen
|
2015-07-09 19:23:14 +00:00
|
|
|
, keyOptions :: Maybe KeyOptions
|
2017-08-15 16:39:10 +00:00
|
|
|
, batchOption :: BatchMode
|
2015-07-09 19:23:14 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
optParser :: CmdParamsDesc -> Parser MoveOptions
|
|
|
|
optParser desc = MoveOptions
|
|
|
|
<$> cmdParams desc
|
2018-04-09 18:29:28 +00:00
|
|
|
<*> parseFromToHereOptions
|
2018-04-13 18:06:25 +00:00
|
|
|
<*> pure RemoveSafe
|
2016-08-03 16:37:12 +00:00
|
|
|
<*> optional (parseKeyOptions <|> parseFailedTransfersOption)
|
2021-08-25 18:20:33 +00:00
|
|
|
<*> parseBatchOption True
|
2015-07-09 19:23:14 +00:00
|
|
|
|
|
|
|
instance DeferredParseClass MoveOptions where
|
|
|
|
finishParse v = MoveOptions
|
|
|
|
<$> pure (moveFiles v)
|
2023-01-18 18:42:39 +00:00
|
|
|
<*> maybe (pure Nothing) (Just <$$> finishParse)
|
|
|
|
(fromToOptions v)
|
2018-04-13 18:06:25 +00:00
|
|
|
<*> pure (removeWhen v)
|
2015-07-09 19:23:14 +00:00
|
|
|
<*> pure (keyOptions v)
|
2017-08-15 16:39:10 +00:00
|
|
|
<*> pure (batchOption v)
|
2015-07-09 19:23:14 +00:00
|
|
|
|
2018-04-13 18:06:25 +00:00
|
|
|
data RemoveWhen = RemoveSafe | RemoveNever
|
2018-04-09 20:09:00 +00:00
|
|
|
deriving (Show, Eq)
|
|
|
|
|
2015-07-09 19:23:14 +00:00
|
|
|
seek :: MoveOptions -> CommandSeek
|
2023-01-18 18:42:39 +00:00
|
|
|
seek o = case fromToOptions o of
|
|
|
|
Just fto -> seek' o fto
|
|
|
|
Nothing -> giveup "Specify --from or --to"
|
|
|
|
|
|
|
|
seek' :: MoveOptions -> FromToHereOptions -> CommandSeek
|
2023-01-24 18:03:26 +00:00
|
|
|
seek' o fto = startConcurrency (stages fto) $ do
|
2017-08-15 16:39:10 +00:00
|
|
|
case batchOption o of
|
2020-07-24 16:05:28 +00:00
|
|
|
NoBatch -> withKeyOptions (keyOptions o) False seeker
|
2021-08-25 18:20:33 +00:00
|
|
|
(commandAction . keyaction)
|
2020-07-13 21:04:02 +00:00
|
|
|
(withFilesInGitAnnex ww seeker)
|
2020-05-28 19:55:17 +00:00
|
|
|
=<< workTreeItems ww (moveFiles o)
|
2022-01-26 16:59:55 +00:00
|
|
|
Batch fmt -> batchOnly (keyOptions o) (moveFiles o) $
|
|
|
|
batchAnnexed fmt seeker keyaction
|
2020-05-26 15:55:50 +00:00
|
|
|
where
|
2020-07-24 16:56:02 +00:00
|
|
|
seeker = AnnexedFileSeeker
|
2023-12-06 17:04:32 +00:00
|
|
|
{ startAction = const $ start fto (removeWhen o)
|
2023-01-18 18:42:39 +00:00
|
|
|
, checkContentPresent = case fto of
|
|
|
|
FromOrToRemote (FromRemote _) -> Nothing
|
|
|
|
FromOrToRemote (ToRemote _) -> Just True
|
|
|
|
ToHere -> Nothing
|
|
|
|
FromRemoteToRemote _ _ -> Nothing
|
copy/move --from-anywhere --to remote
Implementation was simple because it's equivilant to
--from=foo --to remote for each other remote, followed by
--to remote when there's a local copy.
(Or, in the edge case of --from-anywhere --to=here,
it's the same as --to=here.)
Note that, when the local repo does not have a copy,
fromToPerform gets it from a remote, sends it to the destination,
and drops the local copy. Another call to that for a second remote
will notice that the dest now has a copy, and simply drop from the
second remote, avoiding a second transfer.
Also note that, when numcopies doesn't allow dropping it from
everywhere, it will drop it from the cheapest remotes first
(maybe not ideal) up to more expensive remotes, and finally from the local
repo. So the local repo will generally end up holding a copy. Maybe not
ideal in all cases either, but it seems no worse to do that than to end up
with a copy undropped from a remote.
And I'm not entirely happy with the output, eg:
copy bigfile (from r3...) ok
copy bigfile ok
That makes sense if you think of the second line as being
the same as what is output by `git-annex copy bigfile --to bar`,
but it's less clear in this context. Maybe add "(from here...)"?
Also the --json output doesn't have a machine-readable field for
the "from" uuid, and maybe it should?
Sponsored-by: Dartmouth College's DANDI project
2023-11-30 20:32:32 +00:00
|
|
|
FromAnywhereToRemote _ -> Nothing
|
2020-07-24 16:56:02 +00:00
|
|
|
, usesLocationLog = True
|
|
|
|
}
|
2023-01-18 18:42:39 +00:00
|
|
|
keyaction = startKey fto (removeWhen o)
|
2023-04-25 23:26:20 +00:00
|
|
|
ww = WarnUnmatchLsFiles "move"
|
2015-07-09 19:23:14 +00:00
|
|
|
|
2023-01-24 18:03:26 +00:00
|
|
|
stages :: FromToHereOptions -> UsedStages
|
|
|
|
stages (FromOrToRemote (FromRemote _)) = transferStages
|
|
|
|
stages (FromOrToRemote (ToRemote _)) = commandStages
|
|
|
|
stages ToHere = transferStages
|
|
|
|
stages (FromRemoteToRemote _ _) = transferStages
|
copy/move --from-anywhere --to remote
Implementation was simple because it's equivilant to
--from=foo --to remote for each other remote, followed by
--to remote when there's a local copy.
(Or, in the edge case of --from-anywhere --to=here,
it's the same as --to=here.)
Note that, when the local repo does not have a copy,
fromToPerform gets it from a remote, sends it to the destination,
and drops the local copy. Another call to that for a second remote
will notice that the dest now has a copy, and simply drop from the
second remote, avoiding a second transfer.
Also note that, when numcopies doesn't allow dropping it from
everywhere, it will drop it from the cheapest remotes first
(maybe not ideal) up to more expensive remotes, and finally from the local
repo. So the local repo will generally end up holding a copy. Maybe not
ideal in all cases either, but it seems no worse to do that than to end up
with a copy undropped from a remote.
And I'm not entirely happy with the output, eg:
copy bigfile (from r3...) ok
copy bigfile ok
That makes sense if you think of the second line as being
the same as what is output by `git-annex copy bigfile --to bar`,
but it's less clear in this context. Maybe add "(from here...)"?
Also the --json output doesn't have a machine-readable field for
the "from" uuid, and maybe it should?
Sponsored-by: Dartmouth College's DANDI project
2023-11-30 20:32:32 +00:00
|
|
|
stages (FromAnywhereToRemote _) = transferStages
|
2023-01-24 18:03:26 +00:00
|
|
|
|
2020-09-14 20:49:33 +00:00
|
|
|
start :: FromToHereOptions -> RemoveWhen -> SeekInput -> RawFilePath -> Key -> CommandStart
|
|
|
|
start fromto removewhen si f k = start' fromto removewhen afile si k ai
|
2016-07-20 19:22:55 +00:00
|
|
|
where
|
2017-03-10 17:12:24 +00:00
|
|
|
afile = AssociatedFile (Just f)
|
2019-06-06 16:53:24 +00:00
|
|
|
ai = mkActionItem (k, afile)
|
2015-07-09 19:23:14 +00:00
|
|
|
|
2020-09-14 20:49:33 +00:00
|
|
|
startKey :: FromToHereOptions -> RemoveWhen -> (SeekInput, Key, ActionItem) -> CommandStart
|
|
|
|
startKey fromto removewhen (si, k, ai) =
|
|
|
|
start' fromto removewhen (AssociatedFile Nothing) si k ai
|
2015-07-09 19:23:14 +00:00
|
|
|
|
2020-09-14 20:49:33 +00:00
|
|
|
start' :: FromToHereOptions -> RemoveWhen -> AssociatedFile -> SeekInput -> Key -> ActionItem -> CommandStart
|
|
|
|
start' fromto removewhen afile si key ai =
|
2018-04-09 18:38:46 +00:00
|
|
|
case fromto of
|
2023-01-18 18:42:39 +00:00
|
|
|
FromOrToRemote (FromRemote src) ->
|
2017-05-31 20:20:55 +00:00
|
|
|
checkFailedTransferDirection ai Download $
|
2020-09-14 20:49:33 +00:00
|
|
|
fromStart removewhen afile key ai si =<< getParsed src
|
2023-01-18 18:42:39 +00:00
|
|
|
FromOrToRemote (ToRemote dest) ->
|
2017-05-31 20:20:55 +00:00
|
|
|
checkFailedTransferDirection ai Upload $
|
2020-09-14 20:49:33 +00:00
|
|
|
toStart removewhen afile key ai si =<< getParsed dest
|
2023-01-18 18:42:39 +00:00
|
|
|
ToHere ->
|
move --to=here
* move --to=here moves from all reachable remotes to the local repository.
The output of move --from remote is changed slightly, when the remote and
local both have the content. It used to say:
move foo ok
Now:
move foo (from theremote...) ok
That was done so that, when move --to=here is used and the content is
locally present and also in several remotes, it's clear which remotes the
content gets dropped from.
Note that move --to=here will report an error if a non-reachable remote
contains the file, even if the local repository also contains the file. I
think that's reasonable; the user may be intending to move all other copies
of the file from remotes.
OTOH, if a copy of the file is believed to be present in some repository
that is not a configured remote, move --to=here does not report an error.
So a little bit inconsistent, but erroring in this case feels wrong.
copy --to=here came along for free, but it's basically the same behavior as
git-annex get, and probably with not as good messages in edge cases
(especially on failure), so I've not documented it.
This commit was sponsored by Anthony DeRobertis on Patreon.
2017-05-31 20:57:27 +00:00
|
|
|
checkFailedTransferDirection ai Download $
|
2020-09-14 20:49:33 +00:00
|
|
|
toHereStart removewhen afile key ai si
|
2023-01-20 15:10:38 +00:00
|
|
|
FromRemoteToRemote src dest -> do
|
|
|
|
src' <- getParsed src
|
|
|
|
dest' <- getParsed dest
|
|
|
|
fromToStart removewhen afile key ai si src' dest'
|
copy/move --from-anywhere --to remote
Implementation was simple because it's equivilant to
--from=foo --to remote for each other remote, followed by
--to remote when there's a local copy.
(Or, in the edge case of --from-anywhere --to=here,
it's the same as --to=here.)
Note that, when the local repo does not have a copy,
fromToPerform gets it from a remote, sends it to the destination,
and drops the local copy. Another call to that for a second remote
will notice that the dest now has a copy, and simply drop from the
second remote, avoiding a second transfer.
Also note that, when numcopies doesn't allow dropping it from
everywhere, it will drop it from the cheapest remotes first
(maybe not ideal) up to more expensive remotes, and finally from the local
repo. So the local repo will generally end up holding a copy. Maybe not
ideal in all cases either, but it seems no worse to do that than to end up
with a copy undropped from a remote.
And I'm not entirely happy with the output, eg:
copy bigfile (from r3...) ok
copy bigfile ok
That makes sense if you think of the second line as being
the same as what is output by `git-annex copy bigfile --to bar`,
but it's less clear in this context. Maybe add "(from here...)"?
Also the --json output doesn't have a machine-readable field for
the "from" uuid, and maybe it should?
Sponsored-by: Dartmouth College's DANDI project
2023-11-30 20:32:32 +00:00
|
|
|
FromAnywhereToRemote dest -> do
|
|
|
|
dest' <- getParsed dest
|
|
|
|
fromAnywhereToStart removewhen afile key ai si dest'
|
2010-11-02 23:04:24 +00:00
|
|
|
|
make CommandStart return a StartMessage
The goal is to be able to run CommandStart in the main thread when -J is
used, rather than unncessarily passing it off to a worker thread, which
incurs overhead that is signficant when the CommandStart is going to
quickly decide to stop.
To do that, the message it displays needs to be displayed in the worker
thread, after the CommandStart has run.
Also, the change will mean that CommandStart will no longer necessarily
run with the same Annex state as CommandPerform. While its docs already
said it should avoid modifying Annex state, I audited all the
CommandStart code as part of the conversion. (Note that CommandSeek
already sometimes runs with a different Annex state, and that has not been
a source of any problems, so I am not too worried that this change will
lead to breakage going forward.)
The only modification of Annex state I found was it calling
allowMessages in some Commands that default to noMessages. Dealt with
that by adding a startCustomOutput and a startingUsualMessages.
This lets a command start with noMessages and then select the output it
wants for each CommandStart.
One bit of breakage: onlyActionOn has been removed from commands that used it.
The plan is that, since a StartMessage contains an ActionItem,
when a Key can be extracted from that, the parallel job runner can
run onlyActionOn' automatically. Then commands won't need to worry about
this detail. Future work.
Otherwise, this was a fairly straightforward process of making each
CommandStart compile again. Hopefully other behavior changes were mostly
avoided.
In a few cases, a command had a CommandStart that called a CommandPerform
that then called showStart multiple times. I have collapsed those
down to a single start action. The main command to perhaps suffer from it
is Command.Direct, which used to show a start for each file, and no
longer does.
Another minor behavior change is that some commands used showStart
before, but had an associated file and a Key available, so were changed
to ShowStart with an ActionItemAssociatedFile. That will not change the
normal output or behavior, but --json output will now include the key.
This should not break it for anyone using a real json parser.
2019-06-06 19:42:30 +00:00
|
|
|
describeMoveAction :: RemoveWhen -> String
|
|
|
|
describeMoveAction RemoveNever = "copy"
|
|
|
|
describeMoveAction _ = "move"
|
2010-11-27 21:02:53 +00:00
|
|
|
|
2020-09-14 20:49:33 +00:00
|
|
|
toStart :: RemoveWhen -> AssociatedFile -> Key -> ActionItem -> SeekInput -> Remote -> CommandStart
|
|
|
|
toStart removewhen afile key ai si dest = do
|
2011-10-11 18:43:45 +00:00
|
|
|
u <- getUUID
|
2020-07-24 16:56:02 +00:00
|
|
|
if u == Remote.uuid dest
|
|
|
|
then stop
|
2020-09-14 20:49:33 +00:00
|
|
|
else toStart' dest removewhen afile key ai si
|
2014-03-13 18:51:22 +00:00
|
|
|
|
2020-09-14 20:49:33 +00:00
|
|
|
toStart' :: Remote -> RemoveWhen -> AssociatedFile -> Key -> ActionItem -> SeekInput -> CommandStart
|
|
|
|
toStart' dest removewhen afile key ai si = do
|
2022-06-28 19:28:14 +00:00
|
|
|
fast <- Annex.getRead Annex.fast
|
2018-04-09 20:09:00 +00:00
|
|
|
if fast && removewhen == RemoveNever
|
2014-03-13 18:51:22 +00:00
|
|
|
then ifM (expectedPresent dest key)
|
|
|
|
( stop
|
|
|
|
, go True (pure $ Right False)
|
|
|
|
)
|
|
|
|
else go False (Remote.hasKey dest key)
|
|
|
|
where
|
finish CommandStart transition
The hoped for optimisation of CommandStart with -J did not materialize.
In fact, not runnign CommandStart in parallel is slower than -J3.
So, CommandStart are still run in parallel.
(The actual bad performance I've been seeing with -J in my big repo
has to do with building the remoteList.)
But, this is still progress toward making -J faster, because it gets rid
of the onlyActionOn roadblock in the way of making CommandCleanup jobs
run separate from CommandPerform jobs.
Added OnlyActionOn constructor for ActionItem which fixes the
onlyActionOn breakage in the last commit.
Made CustomOutput include an ActionItem, so even things using it can
specify OnlyActionOn.
In Command.Move and Command.Sync, there were CommandStarts that used
includeCommandAction, so output messages, which is no longer allowed.
Fixed by using startingCustomOutput, but that's still not quite right,
since it prevents message display for the includeCommandAction run
inside it too.
2019-06-12 13:23:26 +00:00
|
|
|
go fastcheck isthere =
|
2020-09-14 20:49:33 +00:00
|
|
|
starting (describeMoveAction removewhen) (OnlyActionOn key ai) si $
|
finish CommandStart transition
The hoped for optimisation of CommandStart with -J did not materialize.
In fact, not runnign CommandStart in parallel is slower than -J3.
So, CommandStart are still run in parallel.
(The actual bad performance I've been seeing with -J in my big repo
has to do with building the remoteList.)
But, this is still progress toward making -J faster, because it gets rid
of the onlyActionOn roadblock in the way of making CommandCleanup jobs
run separate from CommandPerform jobs.
Added OnlyActionOn constructor for ActionItem which fixes the
onlyActionOn breakage in the last commit.
Made CustomOutput include an ActionItem, so even things using it can
specify OnlyActionOn.
In Command.Move and Command.Sync, there were CommandStarts that used
includeCommandAction, so output messages, which is no longer allowed.
Fixed by using startingCustomOutput, but that's still not quite right,
since it prevents message display for the includeCommandAction run
inside it too.
2019-06-12 13:23:26 +00:00
|
|
|
toPerform dest removewhen key afile fastcheck =<< isthere
|
2014-03-13 18:51:22 +00:00
|
|
|
|
|
|
|
expectedPresent :: Remote -> Key -> Annex Bool
|
|
|
|
expectedPresent dest key = do
|
2023-11-30 19:11:57 +00:00
|
|
|
remotes <- Remote.keyPossibilities (Remote.IncludeIgnored True) key
|
2014-03-13 18:51:22 +00:00
|
|
|
return $ dest `elem` remotes
|
|
|
|
|
2018-04-09 20:09:00 +00:00
|
|
|
toPerform :: Remote -> RemoveWhen -> Key -> AssociatedFile -> Bool -> Either String Bool -> CommandPerform
|
finishing up move --from --to
Lock the local content for drop after getting it from src, to prevent another
process from using the local content as a copy and dropping it from src,
which would prevent dropping the local content after sending it to dest.
Support resuming an interrupted move that downloaded the content from
src, leaving the local content populated. In this case, the location log
has not been updated to say the content is present locally, so we can
assume that it's resuming and go ahead and drop the local content after
sending it to dest.
Note that if a `git-annex get` is being ran at the same time as a
`git-annex move --from --to`, it may get a file just before the move
processes it. So the location log has not been updated yet, and the move
thinks it's resuming. Resulting in local copy being dropped after it's
sent to the dest. This race is something we'll just have to live with,
it seems.
I also gave up on the idea of checking if the location log had been updated
by a `git-annex get` that is ran at the same time. That wouldn't work, because
the location log is precached in the seek stage, so reading it again after
sending the content to dest would not notice changes made to it, unless the cache
were invalidated, which would slow it down a lot. That idea anyway was subject
to races where it would not detect the concurrent `git-annex get`.
So concurrent `git-annex get` will have results that may be surprising.
To make that less surprising, updated the documentation of this feature to
be explicit that it downloads content to the local repository
temporarily.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 21:07:21 +00:00
|
|
|
toPerform = toPerform' Nothing
|
|
|
|
|
|
|
|
toPerform' :: Maybe ContentRemovalLock -> Remote -> RemoveWhen -> Key -> AssociatedFile -> Bool -> Either String Bool -> CommandPerform
|
|
|
|
toPerform' mcontentlock dest removewhen key afile fastcheck isthere = do
|
2020-10-21 14:31:56 +00:00
|
|
|
srcuuid <- getUUID
|
2010-11-02 23:04:24 +00:00
|
|
|
case isthere of
|
|
|
|
Left err -> do
|
2023-04-10 21:03:41 +00:00
|
|
|
showNote (UnquotedString err)
|
2011-05-15 06:02:46 +00:00
|
|
|
stop
|
2020-10-21 14:31:56 +00:00
|
|
|
Right False -> logMove srcuuid destuuid False key $ \deststartedwithcopy -> do
|
2023-04-10 21:03:41 +00:00
|
|
|
showAction $ UnquotedString $ "to " ++ Remote.name dest
|
2014-03-22 14:42:38 +00:00
|
|
|
ok <- notifyTransfer Upload afile $
|
2020-12-07 18:44:21 +00:00
|
|
|
upload dest key afile stdRetry
|
2010-11-22 21:51:55 +00:00
|
|
|
if ok
|
2020-10-21 14:31:56 +00:00
|
|
|
then finish deststartedwithcopy $
|
2013-02-26 18:39:37 +00:00
|
|
|
Remote.logStatus dest key InfoPresent
|
2011-05-16 17:27:19 +00:00
|
|
|
else do
|
2022-06-09 19:26:25 +00:00
|
|
|
logMoveCleanup deststartedwithcopy
|
2011-05-16 17:27:19 +00:00
|
|
|
when fastcheck $
|
|
|
|
warning "This could have failed because --fast is enabled."
|
|
|
|
stop
|
2020-11-13 18:19:32 +00:00
|
|
|
Right True -> logMove srcuuid destuuid True key $ \deststartedwithcopy ->
|
2020-10-21 14:31:56 +00:00
|
|
|
finish deststartedwithcopy $
|
|
|
|
unlessM (expectedPresent dest key) $
|
|
|
|
Remote.logStatus dest key InfoPresent
|
2012-11-12 05:05:04 +00:00
|
|
|
where
|
2020-10-21 14:31:56 +00:00
|
|
|
destuuid = Remote.uuid dest
|
2018-04-13 18:06:25 +00:00
|
|
|
finish deststartedwithcopy setpresentremote = case removewhen of
|
2018-04-09 20:09:00 +00:00
|
|
|
RemoveNever -> do
|
2016-06-05 17:51:22 +00:00
|
|
|
setpresentremote
|
2022-06-09 19:26:25 +00:00
|
|
|
logMoveCleanup deststartedwithcopy
|
2018-04-09 20:09:00 +00:00
|
|
|
next $ return True
|
finishing up move --from --to
Lock the local content for drop after getting it from src, to prevent another
process from using the local content as a copy and dropping it from src,
which would prevent dropping the local content after sending it to dest.
Support resuming an interrupted move that downloaded the content from
src, leaving the local content populated. In this case, the location log
has not been updated to say the content is present locally, so we can
assume that it's resuming and go ahead and drop the local content after
sending it to dest.
Note that if a `git-annex get` is being ran at the same time as a
`git-annex move --from --to`, it may get a file just before the move
processes it. So the location log has not been updated yet, and the move
thinks it's resuming. Resulting in local copy being dropped after it's
sent to the dest. This race is something we'll just have to live with,
it seems.
I also gave up on the idea of checking if the location log had been updated
by a `git-annex get` that is ran at the same time. That wouldn't work, because
the location log is precached in the seek stage, so reading it again after
sending the content to dest would not notice changes made to it, unless the cache
were invalidated, which would slow it down a lot. That idea anyway was subject
to races where it would not detect the concurrent `git-annex get`.
So concurrent `git-annex get` will have results that may be surprising.
To make that less surprising, updated the documentation of this feature to
be explicit that it downloads content to the local repository
temporarily.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 21:07:21 +00:00
|
|
|
RemoveSafe -> lockcontentforremoval $ \contentlock -> do
|
2018-04-13 18:06:25 +00:00
|
|
|
srcuuid <- getUUID
|
2022-06-09 19:26:25 +00:00
|
|
|
r <- willDropMakeItWorse srcuuid destuuid deststartedwithcopy key afile >>= \case
|
2018-04-13 18:06:25 +00:00
|
|
|
DropAllowed -> drophere setpresentremote contentlock "moved"
|
|
|
|
DropCheckNumCopies -> do
|
2021-06-15 15:38:44 +00:00
|
|
|
(numcopies, mincopies) <- getSafestNumMinCopies afile key
|
2018-04-13 18:06:25 +00:00
|
|
|
(tocheck, verified) <- verifiableCopies key [srcuuid]
|
2024-06-23 13:28:18 +00:00
|
|
|
verifyEnoughCopiesToDrop "" key (Just srcuuid) (Just contentlock)
|
2021-01-06 18:11:08 +00:00
|
|
|
numcopies mincopies [srcuuid] verified
|
2018-04-13 18:06:25 +00:00
|
|
|
(UnVerifiedRemote dest : tocheck)
|
|
|
|
(drophere setpresentremote contentlock . showproof)
|
|
|
|
(faileddrophere setpresentremote)
|
|
|
|
DropWorse -> faileddrophere setpresentremote
|
2022-06-09 19:26:25 +00:00
|
|
|
logMoveCleanup deststartedwithcopy
|
|
|
|
return r
|
2018-04-13 18:06:25 +00:00
|
|
|
showproof proof = "proof: " ++ show proof
|
|
|
|
drophere setpresentremote contentlock reason = do
|
2021-04-06 19:41:24 +00:00
|
|
|
fastDebug "Command.Move" $ unwords
|
2018-04-13 18:06:25 +00:00
|
|
|
[ "Dropping from here"
|
|
|
|
, "(" ++ reason ++ ")"
|
|
|
|
]
|
|
|
|
-- Drop content before updating location logs,
|
|
|
|
-- in case disk space is very low this frees
|
|
|
|
-- up space before writing data to disk.
|
|
|
|
removeAnnex contentlock
|
|
|
|
next $ do
|
|
|
|
() <- setpresentremote
|
2021-06-25 19:22:05 +00:00
|
|
|
Command.Drop.cleanupLocal key (Command.Drop.DroppingUnused False)
|
2018-04-13 18:06:25 +00:00
|
|
|
faileddrophere setpresentremote = do
|
|
|
|
showLongNote "(Use --force to override this check, or adjust numcopies.)"
|
|
|
|
showLongNote "Content not dropped from here."
|
|
|
|
next $ do
|
|
|
|
() <- setpresentremote
|
|
|
|
return False
|
2020-07-25 15:54:34 +00:00
|
|
|
|
finishing up move --from --to
Lock the local content for drop after getting it from src, to prevent another
process from using the local content as a copy and dropping it from src,
which would prevent dropping the local content after sending it to dest.
Support resuming an interrupted move that downloaded the content from
src, leaving the local content populated. In this case, the location log
has not been updated to say the content is present locally, so we can
assume that it's resuming and go ahead and drop the local content after
sending it to dest.
Note that if a `git-annex get` is being ran at the same time as a
`git-annex move --from --to`, it may get a file just before the move
processes it. So the location log has not been updated yet, and the move
thinks it's resuming. Resulting in local copy being dropped after it's
sent to the dest. This race is something we'll just have to live with,
it seems.
I also gave up on the idea of checking if the location log had been updated
by a `git-annex get` that is ran at the same time. That wouldn't work, because
the location log is precached in the seek stage, so reading it again after
sending the content to dest would not notice changes made to it, unless the cache
were invalidated, which would slow it down a lot. That idea anyway was subject
to races where it would not detect the concurrent `git-annex get`.
So concurrent `git-annex get` will have results that may be surprising.
To make that less surprising, updated the documentation of this feature to
be explicit that it downloads content to the local repository
temporarily.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 21:07:21 +00:00
|
|
|
lockcontentforremoval a = case mcontentlock of
|
|
|
|
Nothing -> lockContentForRemoval key lockfailed a
|
|
|
|
Just contentlock -> a contentlock
|
|
|
|
|
2020-07-25 15:54:34 +00:00
|
|
|
-- This occurs when, for example, two files are being dropped
|
|
|
|
-- and have the same content. The seek stage checks if the content
|
|
|
|
-- is present, but due to buffering, may find it present for the
|
|
|
|
-- second file before the first is dropped. If so, nothing remains
|
|
|
|
-- to be done except for cleaning up.
|
2021-06-25 19:22:05 +00:00
|
|
|
lockfailed = next $ Command.Drop.cleanupLocal key (Command.Drop.DroppingUnused False)
|
2010-11-02 23:04:24 +00:00
|
|
|
|
2020-09-14 20:49:33 +00:00
|
|
|
fromStart :: RemoveWhen -> AssociatedFile -> Key -> ActionItem -> SeekInput -> Remote -> CommandStart
|
|
|
|
fromStart removewhen afile key ai si src =
|
2020-07-24 16:56:02 +00:00
|
|
|
stopUnless (fromOk src key) $
|
2020-09-14 20:49:33 +00:00
|
|
|
starting (describeMoveAction removewhen) (OnlyActionOn key ai) si $
|
make CommandStart return a StartMessage
The goal is to be able to run CommandStart in the main thread when -J is
used, rather than unncessarily passing it off to a worker thread, which
incurs overhead that is signficant when the CommandStart is going to
quickly decide to stop.
To do that, the message it displays needs to be displayed in the worker
thread, after the CommandStart has run.
Also, the change will mean that CommandStart will no longer necessarily
run with the same Annex state as CommandPerform. While its docs already
said it should avoid modifying Annex state, I audited all the
CommandStart code as part of the conversion. (Note that CommandSeek
already sometimes runs with a different Annex state, and that has not been
a source of any problems, so I am not too worried that this change will
lead to breakage going forward.)
The only modification of Annex state I found was it calling
allowMessages in some Commands that default to noMessages. Dealt with
that by adding a startCustomOutput and a startingUsualMessages.
This lets a command start with noMessages and then select the output it
wants for each CommandStart.
One bit of breakage: onlyActionOn has been removed from commands that used it.
The plan is that, since a StartMessage contains an ActionItem,
when a Key can be extracted from that, the parallel job runner can
run onlyActionOn' automatically. Then commands won't need to worry about
this detail. Future work.
Otherwise, this was a fairly straightforward process of making each
CommandStart compile again. Hopefully other behavior changes were mostly
avoided.
In a few cases, a command had a CommandStart that called a CommandPerform
that then called showStart multiple times. I have collapsed those
down to a single start action. The main command to perhaps suffer from it
is Command.Direct, which used to show a start for each file, and no
longer does.
Another minor behavior change is that some commands used showStart
before, but had an associated file and a Key available, so were changed
to ShowStart with an ActionItemAssociatedFile. That will not change the
normal output or behavior, but --json output will now include the key.
This should not break it for anyone using a real json parser.
2019-06-06 19:42:30 +00:00
|
|
|
fromPerform src removewhen key afile
|
2012-11-12 05:05:04 +00:00
|
|
|
|
2011-12-31 08:11:39 +00:00
|
|
|
fromOk :: Remote -> Key -> Annex Bool
|
check Remote.hasKeyCheap again
In cd1676d604, it stopped using that to avoid surprising behavior
when the location log and remote content were out of sync.
But, it seems that may have changed some behavior users relied on as
well, and also Remote.hasKeyCheap should be faster than checking then
location log.
So, try Remote.hasKeyCheap first, and only if it does not have the key,
fall back to checking the location log. If the location log still thinks
it's present, go ahead and try to get it, so the user will see a failure
rather than silently skipping a file what whereis says is on the remote.
This does make slightly slower the case where the remote does not have
the key, and location log and Remote.hasKeyCheap agree, since it now
checks both. But only 1 stat slower.
2020-12-15 18:44:00 +00:00
|
|
|
fromOk src key
|
|
|
|
-- check if the remote contains the key, when it can be done cheaply
|
|
|
|
| Remote.hasKeyCheap src =
|
2020-12-15 20:35:06 +00:00
|
|
|
Remote.hasKey src key >>= \case
|
check Remote.hasKeyCheap again
In cd1676d604, it stopped using that to avoid surprising behavior
when the location log and remote content were out of sync.
But, it seems that may have changed some behavior users relied on as
well, and also Remote.hasKeyCheap should be faster than checking then
location log.
So, try Remote.hasKeyCheap first, and only if it does not have the key,
fall back to checking the location log. If the location log still thinks
it's present, go ahead and try to get it, so the user will see a failure
rather than silently skipping a file what whereis says is on the remote.
This does make slightly slower the case where the remote does not have
the key, and location log and Remote.hasKeyCheap agree, since it now
checks both. But only 1 stat slower.
2020-12-15 18:44:00 +00:00
|
|
|
Right True -> return True
|
|
|
|
-- Don't skip getting the key just because the
|
|
|
|
-- remote no longer contains it if the log
|
|
|
|
-- says the remote is supposed to contain it;
|
|
|
|
-- that would be surprising behavior.
|
|
|
|
_ -> checklog
|
|
|
|
| otherwise = checklog
|
|
|
|
where
|
|
|
|
checklog = do
|
|
|
|
u <- getUUID
|
2023-11-30 19:11:57 +00:00
|
|
|
remotes <- Remote.keyPossibilities (Remote.IncludeIgnored True) key
|
check Remote.hasKeyCheap again
In cd1676d604, it stopped using that to avoid surprising behavior
when the location log and remote content were out of sync.
But, it seems that may have changed some behavior users relied on as
well, and also Remote.hasKeyCheap should be faster than checking then
location log.
So, try Remote.hasKeyCheap first, and only if it does not have the key,
fall back to checking the location log. If the location log still thinks
it's present, go ahead and try to get it, so the user will see a failure
rather than silently skipping a file what whereis says is on the remote.
This does make slightly slower the case where the remote does not have
the key, and location log and Remote.hasKeyCheap agree, since it now
checks both. But only 1 stat slower.
2020-12-15 18:44:00 +00:00
|
|
|
return $ u /= Remote.uuid src && elem src remotes
|
2012-11-12 05:05:04 +00:00
|
|
|
|
2018-04-09 20:09:00 +00:00
|
|
|
fromPerform :: Remote -> RemoveWhen -> Key -> AssociatedFile -> CommandPerform
|
|
|
|
fromPerform src removewhen key afile = do
|
2020-10-21 14:31:56 +00:00
|
|
|
present <- inAnnex key
|
fully working move --from --to (not release quality)
When the destination already has a copy, it behaves the same as
drop --from really, but display it as a move and implement it
reusing the factored out code from fromPerform.
(Note that willDropMakeItWorse never returns DropAllowed in that
situation, because it's told that dest has a copy. So numcopies is
always checked.)
And when only the source and not the local repo or destination have a
copy, do the full copy from source to local, then copy from local to
dest, then drop from local, then drop from source dance.
This is complicated by fromPerform being hardcoded to assume there is a
local copy, but the local copy has already been dropped. That's why
it uses cleanupfromsrc RemoveNever to avoid the code that makes that
assumption, and finishes with a call to dropfromsrc.
And, since the location log has not yet been updated, checking numcopies
was not working, until I added UnVerifiedRemote dest to the list of
things to check.
This is not yet quite mergeable though. There are two things in the
comment above fromToPerform that are not implemented yet: Checking the
location log before dropping the local copy, and locking the temporary
local copy for drop.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 19:36:06 +00:00
|
|
|
finish <- fromPerform' present True src key afile
|
|
|
|
finish removewhen
|
2023-01-23 17:16:14 +00:00
|
|
|
|
fully working move --from --to (not release quality)
When the destination already has a copy, it behaves the same as
drop --from really, but display it as a move and implement it
reusing the factored out code from fromPerform.
(Note that willDropMakeItWorse never returns DropAllowed in that
situation, because it's told that dest has a copy. So numcopies is
always checked.)
And when only the source and not the local repo or destination have a
copy, do the full copy from source to local, then copy from local to
dest, then drop from local, then drop from source dance.
This is complicated by fromPerform being hardcoded to assume there is a
local copy, but the local copy has already been dropped. That's why
it uses cleanupfromsrc RemoveNever to avoid the code that makes that
assumption, and finishes with a call to dropfromsrc.
And, since the location log has not yet been updated, checking numcopies
was not working, until I added UnVerifiedRemote dest to the list of
things to check.
This is not yet quite mergeable though. There are two things in the
comment above fromToPerform that are not implemented yet: Checking the
location log before dropping the local copy, and locking the temporary
local copy for drop.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 19:36:06 +00:00
|
|
|
fromPerform' :: Bool -> Bool -> Remote -> Key -> AssociatedFile -> Annex (RemoveWhen -> CommandPerform)
|
|
|
|
fromPerform' present updatelocationlog src key afile = do
|
2023-04-10 21:03:41 +00:00
|
|
|
showAction $ UnquotedString $ "from " ++ Remote.name src
|
2020-10-21 14:31:56 +00:00
|
|
|
destuuid <- getUUID
|
fully working move --from --to (not release quality)
When the destination already has a copy, it behaves the same as
drop --from really, but display it as a move and implement it
reusing the factored out code from fromPerform.
(Note that willDropMakeItWorse never returns DropAllowed in that
situation, because it's told that dest has a copy. So numcopies is
always checked.)
And when only the source and not the local repo or destination have a
copy, do the full copy from source to local, then copy from local to
dest, then drop from local, then drop from source dance.
This is complicated by fromPerform being hardcoded to assume there is a
local copy, but the local copy has already been dropped. That's why
it uses cleanupfromsrc RemoveNever to avoid the code that makes that
assumption, and finishes with a call to dropfromsrc.
And, since the location log has not yet been updated, checking numcopies
was not working, until I added UnVerifiedRemote dest to the list of
things to check.
This is not yet quite mergeable though. There are two things in the
comment above fromToPerform that are not implemented yet: Checking the
location log before dropping the local copy, and locking the temporary
local copy for drop.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 19:36:06 +00:00
|
|
|
logMove (Remote.uuid src) destuuid present key $ \deststartedwithcopy ->
|
2020-10-21 14:31:56 +00:00
|
|
|
if present
|
fully working move --from --to (not release quality)
When the destination already has a copy, it behaves the same as
drop --from really, but display it as a move and implement it
reusing the factored out code from fromPerform.
(Note that willDropMakeItWorse never returns DropAllowed in that
situation, because it's told that dest has a copy. So numcopies is
always checked.)
And when only the source and not the local repo or destination have a
copy, do the full copy from source to local, then copy from local to
dest, then drop from local, then drop from source dance.
This is complicated by fromPerform being hardcoded to assume there is a
local copy, but the local copy has already been dropped. That's why
it uses cleanupfromsrc RemoveNever to avoid the code that makes that
assumption, and finishes with a call to dropfromsrc.
And, since the location log has not yet been updated, checking numcopies
was not working, until I added UnVerifiedRemote dest to the list of
things to check.
This is not yet quite mergeable though. There are two things in the
comment above fromToPerform that are not implemented yet: Checking the
location log before dropping the local copy, and locking the temporary
local copy for drop.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 19:36:06 +00:00
|
|
|
then return $ finish deststartedwithcopy True
|
|
|
|
else do
|
|
|
|
got <- get
|
|
|
|
return $ finish deststartedwithcopy got
|
2012-11-12 05:05:04 +00:00
|
|
|
where
|
2020-12-07 18:44:21 +00:00
|
|
|
get = notifyTransfer Download afile $
|
fully working move --from --to (not release quality)
When the destination already has a copy, it behaves the same as
drop --from really, but display it as a move and implement it
reusing the factored out code from fromPerform.
(Note that willDropMakeItWorse never returns DropAllowed in that
situation, because it's told that dest has a copy. So numcopies is
always checked.)
And when only the source and not the local repo or destination have a
copy, do the full copy from source to local, then copy from local to
dest, then drop from local, then drop from source dance.
This is complicated by fromPerform being hardcoded to assume there is a
local copy, but the local copy has already been dropped. That's why
it uses cleanupfromsrc RemoveNever to avoid the code that makes that
assumption, and finishes with a call to dropfromsrc.
And, since the location log has not yet been updated, checking numcopies
was not working, until I added UnVerifiedRemote dest to the list of
things to check.
This is not yet quite mergeable though. There are two things in the
comment above fromToPerform that are not implemented yet: Checking the
location log before dropping the local copy, and locking the temporary
local copy for drop.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 19:36:06 +00:00
|
|
|
logdownload .
|
2023-01-23 17:45:26 +00:00
|
|
|
download src key afile stdRetry
|
2020-10-21 14:31:56 +00:00
|
|
|
|
fully working move --from --to (not release quality)
When the destination already has a copy, it behaves the same as
drop --from really, but display it as a move and implement it
reusing the factored out code from fromPerform.
(Note that willDropMakeItWorse never returns DropAllowed in that
situation, because it's told that dest has a copy. So numcopies is
always checked.)
And when only the source and not the local repo or destination have a
copy, do the full copy from source to local, then copy from local to
dest, then drop from local, then drop from source dance.
This is complicated by fromPerform being hardcoded to assume there is a
local copy, but the local copy has already been dropped. That's why
it uses cleanupfromsrc RemoveNever to avoid the code that makes that
assumption, and finishes with a call to dropfromsrc.
And, since the location log has not yet been updated, checking numcopies
was not working, until I added UnVerifiedRemote dest to the list of
things to check.
This is not yet quite mergeable though. There are two things in the
comment above fromToPerform that are not implemented yet: Checking the
location log before dropping the local copy, and locking the temporary
local copy for drop.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 19:36:06 +00:00
|
|
|
logdownload a
|
|
|
|
| updatelocationlog = logStatusAfter key a
|
|
|
|
| otherwise = a
|
|
|
|
|
|
|
|
finish deststartedwithcopy False _ = do
|
2022-06-09 19:26:25 +00:00
|
|
|
logMoveCleanup deststartedwithcopy
|
|
|
|
stop -- copy failed
|
fully working move --from --to (not release quality)
When the destination already has a copy, it behaves the same as
drop --from really, but display it as a move and implement it
reusing the factored out code from fromPerform.
(Note that willDropMakeItWorse never returns DropAllowed in that
situation, because it's told that dest has a copy. So numcopies is
always checked.)
And when only the source and not the local repo or destination have a
copy, do the full copy from source to local, then copy from local to
dest, then drop from local, then drop from source dance.
This is complicated by fromPerform being hardcoded to assume there is a
local copy, but the local copy has already been dropped. That's why
it uses cleanupfromsrc RemoveNever to avoid the code that makes that
assumption, and finishes with a call to dropfromsrc.
And, since the location log has not yet been updated, checking numcopies
was not working, until I added UnVerifiedRemote dest to the list of
things to check.
This is not yet quite mergeable though. There are two things in the
comment above fromToPerform that are not implemented yet: Checking the
location log before dropping the local copy, and locking the temporary
local copy for drop.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 19:36:06 +00:00
|
|
|
finish deststartedwithcopy True RemoveNever = do
|
2022-06-09 19:26:25 +00:00
|
|
|
logMoveCleanup deststartedwithcopy
|
|
|
|
next $ return True -- copy complete
|
fully working move --from --to (not release quality)
When the destination already has a copy, it behaves the same as
drop --from really, but display it as a move and implement it
reusing the factored out code from fromPerform.
(Note that willDropMakeItWorse never returns DropAllowed in that
situation, because it's told that dest has a copy. So numcopies is
always checked.)
And when only the source and not the local repo or destination have a
copy, do the full copy from source to local, then copy from local to
dest, then drop from local, then drop from source dance.
This is complicated by fromPerform being hardcoded to assume there is a
local copy, but the local copy has already been dropped. That's why
it uses cleanupfromsrc RemoveNever to avoid the code that makes that
assumption, and finishes with a call to dropfromsrc.
And, since the location log has not yet been updated, checking numcopies
was not working, until I added UnVerifiedRemote dest to the list of
things to check.
This is not yet quite mergeable though. There are two things in the
comment above fromToPerform that are not implemented yet: Checking the
location log before dropping the local copy, and locking the temporary
local copy for drop.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 19:36:06 +00:00
|
|
|
finish deststartedwithcopy True RemoveSafe = do
|
2018-04-13 18:06:25 +00:00
|
|
|
destuuid <- getUUID
|
add content retention files
This allows lockContentShared to lock content for eg, 10 minutes and
if the process then gets terminated before it can unlock, the content
will remain locked for that amount of time.
The Windows implementation is not yet tested.
In P2P.Annex, a duration of 10 minutes is used. This way, when p2pstdio
or remotedaemon is serving the P2P protocol, and is asked to
LOCKCONTENT, and that process gets killed, the content will not be
subject to deletion. This is not a perfect solution to
doc/todo/P2P_locking_connection_drop_safety.mdwn yet, but it gets most
of the way there, without needing any P2P protocol changes.
This is only done in v10 and higher repositories (or on Windows). It
might be possible to backport it to v8 or earlier, but it would
complicate locking even further, and without a separate lock file, might
be hard. I think that by the time this fix reaches a given user, they
will probably have been running git-annex 10.x long enough that their v8
repositories will have upgraded to v10 after the 1 year wait. And it's
not as if git-annex hasn't already been subject to this problem (though
I have not heard of any data loss caused by it) for 6 years already, so
waiting another fraction of a year on top of however long it takes this
fix to reach users is unlikely to be a problem.
2024-07-03 18:44:38 +00:00
|
|
|
lockContentShared key Nothing $ \_lck ->
|
fully working move --from --to (not release quality)
When the destination already has a copy, it behaves the same as
drop --from really, but display it as a move and implement it
reusing the factored out code from fromPerform.
(Note that willDropMakeItWorse never returns DropAllowed in that
situation, because it's told that dest has a copy. So numcopies is
always checked.)
And when only the source and not the local repo or destination have a
copy, do the full copy from source to local, then copy from local to
dest, then drop from local, then drop from source dance.
This is complicated by fromPerform being hardcoded to assume there is a
local copy, but the local copy has already been dropped. That's why
it uses cleanupfromsrc RemoveNever to avoid the code that makes that
assumption, and finishes with a call to dropfromsrc.
And, since the location log has not yet been updated, checking numcopies
was not working, until I added UnVerifiedRemote dest to the list of
things to check.
This is not yet quite mergeable though. There are two things in the
comment above fromToPerform that are not implemented yet: Checking the
location log before dropping the local copy, and locking the temporary
local copy for drop.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 19:36:06 +00:00
|
|
|
fromDrop src destuuid deststartedwithcopy key afile id
|
|
|
|
|
|
|
|
fromDrop :: Remote -> UUID -> DestStartedWithCopy -> Key -> AssociatedFile -> ([UnVerifiedCopy] -> [UnVerifiedCopy])-> CommandPerform
|
|
|
|
fromDrop src destuuid deststartedwithcopy key afile adjusttocheck =
|
|
|
|
willDropMakeItWorse (Remote.uuid src) destuuid deststartedwithcopy key afile >>= \case
|
toward SafeDropProof expiry checking
Added Maybe POSIXTime to SafeDropProof, which gets set when the proof is
based on a LockedCopy. If there are several LockedCopies, it uses the
closest expiry time. That is not optimal, it may be that the proof
expires based on one LockedCopy but another one has not expired. But
that seems unlikely to really happen, and anyway the user can just
re-run a drop if it fails due to expiry.
Pass the SafeDropProof to removeKey, which is responsible for checking
it for expiry in situations where that could be a problem. Which really
only means in Remote.Git.
Made Remote.Git check expiry when dropping from a local remote.
Checking expiry when dropping from a P2P remote is not yet implemented.
P2P.Protocol.remove has SafeDropProof plumbed through to it for that
purpose.
Fixing the remaining 2 build warnings should complete this work.
Note that the use of a POSIXTime here means that if the clock gets set
forward while git-annex is in the middle of a drop, it may say that
dropping took too long. That seems ok. Less ok is that if the clock gets
turned back a sufficient amount (eg 5 minutes), proof expiry won't be
noticed. It might be better to use the Monotonic clock, but that doesn't
advance when a laptop is suspended, and while there is the linux
Boottime clock, that is not available on other systems. Perhaps a
combination of POSIXTime and the Monotonic clock could detect laptop
suspension and also detect clock being turned back?
There is a potential future flag day where
p2pDefaultLockContentRetentionDuration is not assumed, but is probed
using the P2P protocol, and peers that don't support it can no longer
produce a LockedCopy. Until that happens, when git-annex is
communicating with older peers there is a risk of data loss when
a ssh connection closes during LOCKCONTENT.
2024-07-04 16:23:46 +00:00
|
|
|
DropAllowed -> dropremote Nothing "moved"
|
fully working move --from --to (not release quality)
When the destination already has a copy, it behaves the same as
drop --from really, but display it as a move and implement it
reusing the factored out code from fromPerform.
(Note that willDropMakeItWorse never returns DropAllowed in that
situation, because it's told that dest has a copy. So numcopies is
always checked.)
And when only the source and not the local repo or destination have a
copy, do the full copy from source to local, then copy from local to
dest, then drop from local, then drop from source dance.
This is complicated by fromPerform being hardcoded to assume there is a
local copy, but the local copy has already been dropped. That's why
it uses cleanupfromsrc RemoveNever to avoid the code that makes that
assumption, and finishes with a call to dropfromsrc.
And, since the location log has not yet been updated, checking numcopies
was not working, until I added UnVerifiedRemote dest to the list of
things to check.
This is not yet quite mergeable though. There are two things in the
comment above fromToPerform that are not implemented yet: Checking the
location log before dropping the local copy, and locking the temporary
local copy for drop.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 19:36:06 +00:00
|
|
|
DropCheckNumCopies -> do
|
|
|
|
(numcopies, mincopies) <- getSafestNumMinCopies afile key
|
|
|
|
(tocheck, verified) <- verifiableCopies key [Remote.uuid src]
|
2024-06-23 13:28:18 +00:00
|
|
|
verifyEnoughCopiesToDrop "" key (Just (Remote.uuid src)) Nothing numcopies mincopies [Remote.uuid src] verified
|
toward SafeDropProof expiry checking
Added Maybe POSIXTime to SafeDropProof, which gets set when the proof is
based on a LockedCopy. If there are several LockedCopies, it uses the
closest expiry time. That is not optimal, it may be that the proof
expires based on one LockedCopy but another one has not expired. But
that seems unlikely to really happen, and anyway the user can just
re-run a drop if it fails due to expiry.
Pass the SafeDropProof to removeKey, which is responsible for checking
it for expiry in situations where that could be a problem. Which really
only means in Remote.Git.
Made Remote.Git check expiry when dropping from a local remote.
Checking expiry when dropping from a P2P remote is not yet implemented.
P2P.Protocol.remove has SafeDropProof plumbed through to it for that
purpose.
Fixing the remaining 2 build warnings should complete this work.
Note that the use of a POSIXTime here means that if the clock gets set
forward while git-annex is in the middle of a drop, it may say that
dropping took too long. That seems ok. Less ok is that if the clock gets
turned back a sufficient amount (eg 5 minutes), proof expiry won't be
noticed. It might be better to use the Monotonic clock, but that doesn't
advance when a laptop is suspended, and while there is the linux
Boottime clock, that is not available on other systems. Perhaps a
combination of POSIXTime and the Monotonic clock could detect laptop
suspension and also detect clock being turned back?
There is a potential future flag day where
p2pDefaultLockContentRetentionDuration is not assumed, but is probed
using the P2P protocol, and peers that don't support it can no longer
produce a LockedCopy. Until that happens, when git-annex is
communicating with older peers there is a risk of data loss when
a ssh connection closes during LOCKCONTENT.
2024-07-04 16:23:46 +00:00
|
|
|
(adjusttocheck tocheck) dropremotewithproof faileddropremote
|
fully working move --from --to (not release quality)
When the destination already has a copy, it behaves the same as
drop --from really, but display it as a move and implement it
reusing the factored out code from fromPerform.
(Note that willDropMakeItWorse never returns DropAllowed in that
situation, because it's told that dest has a copy. So numcopies is
always checked.)
And when only the source and not the local repo or destination have a
copy, do the full copy from source to local, then copy from local to
dest, then drop from local, then drop from source dance.
This is complicated by fromPerform being hardcoded to assume there is a
local copy, but the local copy has already been dropped. That's why
it uses cleanupfromsrc RemoveNever to avoid the code that makes that
assumption, and finishes with a call to dropfromsrc.
And, since the location log has not yet been updated, checking numcopies
was not working, until I added UnVerifiedRemote dest to the list of
things to check.
This is not yet quite mergeable though. There are two things in the
comment above fromToPerform that are not implemented yet: Checking the
location log before dropping the local copy, and locking the temporary
local copy for drop.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 19:36:06 +00:00
|
|
|
DropWorse -> faileddropremote
|
|
|
|
where
|
2018-04-13 18:06:25 +00:00
|
|
|
showproof proof = "proof: " ++ show proof
|
fully working move --from --to (not release quality)
When the destination already has a copy, it behaves the same as
drop --from really, but display it as a move and implement it
reusing the factored out code from fromPerform.
(Note that willDropMakeItWorse never returns DropAllowed in that
situation, because it's told that dest has a copy. So numcopies is
always checked.)
And when only the source and not the local repo or destination have a
copy, do the full copy from source to local, then copy from local to
dest, then drop from local, then drop from source dance.
This is complicated by fromPerform being hardcoded to assume there is a
local copy, but the local copy has already been dropped. That's why
it uses cleanupfromsrc RemoveNever to avoid the code that makes that
assumption, and finishes with a call to dropfromsrc.
And, since the location log has not yet been updated, checking numcopies
was not working, until I added UnVerifiedRemote dest to the list of
things to check.
This is not yet quite mergeable though. There are two things in the
comment above fromToPerform that are not implemented yet: Checking the
location log before dropping the local copy, and locking the temporary
local copy for drop.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 19:36:06 +00:00
|
|
|
|
toward SafeDropProof expiry checking
Added Maybe POSIXTime to SafeDropProof, which gets set when the proof is
based on a LockedCopy. If there are several LockedCopies, it uses the
closest expiry time. That is not optimal, it may be that the proof
expires based on one LockedCopy but another one has not expired. But
that seems unlikely to really happen, and anyway the user can just
re-run a drop if it fails due to expiry.
Pass the SafeDropProof to removeKey, which is responsible for checking
it for expiry in situations where that could be a problem. Which really
only means in Remote.Git.
Made Remote.Git check expiry when dropping from a local remote.
Checking expiry when dropping from a P2P remote is not yet implemented.
P2P.Protocol.remove has SafeDropProof plumbed through to it for that
purpose.
Fixing the remaining 2 build warnings should complete this work.
Note that the use of a POSIXTime here means that if the clock gets set
forward while git-annex is in the middle of a drop, it may say that
dropping took too long. That seems ok. Less ok is that if the clock gets
turned back a sufficient amount (eg 5 minutes), proof expiry won't be
noticed. It might be better to use the Monotonic clock, but that doesn't
advance when a laptop is suspended, and while there is the linux
Boottime clock, that is not available on other systems. Perhaps a
combination of POSIXTime and the Monotonic clock could detect laptop
suspension and also detect clock being turned back?
There is a potential future flag day where
p2pDefaultLockContentRetentionDuration is not assumed, but is probed
using the P2P protocol, and peers that don't support it can no longer
produce a LockedCopy. Until that happens, when git-annex is
communicating with older peers there is a risk of data loss when
a ssh connection closes during LOCKCONTENT.
2024-07-04 16:23:46 +00:00
|
|
|
dropremotewithproof proof =
|
|
|
|
dropremote (Just proof) (showproof proof)
|
|
|
|
|
|
|
|
dropremote mproof reason = do
|
2021-04-06 19:41:24 +00:00
|
|
|
fastDebug "Command.Move" $ unwords
|
2015-10-09 20:16:03 +00:00
|
|
|
[ "Dropping from remote"
|
|
|
|
, show src
|
2018-04-13 18:06:25 +00:00
|
|
|
, "(" ++ reason ++ ")"
|
2015-10-09 20:16:03 +00:00
|
|
|
]
|
toward SafeDropProof expiry checking
Added Maybe POSIXTime to SafeDropProof, which gets set when the proof is
based on a LockedCopy. If there are several LockedCopies, it uses the
closest expiry time. That is not optimal, it may be that the proof
expires based on one LockedCopy but another one has not expired. But
that seems unlikely to really happen, and anyway the user can just
re-run a drop if it fails due to expiry.
Pass the SafeDropProof to removeKey, which is responsible for checking
it for expiry in situations where that could be a problem. Which really
only means in Remote.Git.
Made Remote.Git check expiry when dropping from a local remote.
Checking expiry when dropping from a P2P remote is not yet implemented.
P2P.Protocol.remove has SafeDropProof plumbed through to it for that
purpose.
Fixing the remaining 2 build warnings should complete this work.
Note that the use of a POSIXTime here means that if the clock gets set
forward while git-annex is in the middle of a drop, it may say that
dropping took too long. That seems ok. Less ok is that if the clock gets
turned back a sufficient amount (eg 5 minutes), proof expiry won't be
noticed. It might be better to use the Monotonic clock, but that doesn't
advance when a laptop is suspended, and while there is the linux
Boottime clock, that is not available on other systems. Perhaps a
combination of POSIXTime and the Monotonic clock could detect laptop
suspension and also detect clock being turned back?
There is a potential future flag day where
p2pDefaultLockContentRetentionDuration is not assumed, but is probed
using the P2P protocol, and peers that don't support it can no longer
produce a LockedCopy. Until that happens, when git-annex is
communicating with older peers there is a risk of data loss when
a ssh connection closes during LOCKCONTENT.
2024-07-04 16:23:46 +00:00
|
|
|
ok <- Remote.action (Remote.removeKey src mproof key)
|
2022-06-09 19:26:25 +00:00
|
|
|
when ok $
|
|
|
|
logMoveCleanup deststartedwithcopy
|
2021-06-25 19:22:05 +00:00
|
|
|
next $ Command.Drop.cleanupRemote key src (Command.Drop.DroppingUnused False) ok
|
fully working move --from --to (not release quality)
When the destination already has a copy, it behaves the same as
drop --from really, but display it as a move and implement it
reusing the factored out code from fromPerform.
(Note that willDropMakeItWorse never returns DropAllowed in that
situation, because it's told that dest has a copy. So numcopies is
always checked.)
And when only the source and not the local repo or destination have a
copy, do the full copy from source to local, then copy from local to
dest, then drop from local, then drop from source dance.
This is complicated by fromPerform being hardcoded to assume there is a
local copy, but the local copy has already been dropped. That's why
it uses cleanupfromsrc RemoveNever to avoid the code that makes that
assumption, and finishes with a call to dropfromsrc.
And, since the location log has not yet been updated, checking numcopies
was not working, until I added UnVerifiedRemote dest to the list of
things to check.
This is not yet quite mergeable though. There are two things in the
comment above fromToPerform that are not implemented yet: Checking the
location log before dropping the local copy, and locking the temporary
local copy for drop.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 19:36:06 +00:00
|
|
|
|
|
|
|
faileddropremote = do
|
2018-04-13 18:06:25 +00:00
|
|
|
showLongNote "(Use --force to override this check, or adjust numcopies.)"
|
2023-04-10 21:03:41 +00:00
|
|
|
showLongNote $ UnquotedString $ "Content not dropped from " ++ Remote.name src ++ "."
|
2022-06-09 19:26:25 +00:00
|
|
|
logMoveCleanup deststartedwithcopy
|
2018-04-13 18:06:25 +00:00
|
|
|
next $ return False
|
move --to=here
* move --to=here moves from all reachable remotes to the local repository.
The output of move --from remote is changed slightly, when the remote and
local both have the content. It used to say:
move foo ok
Now:
move foo (from theremote...) ok
That was done so that, when move --to=here is used and the content is
locally present and also in several remotes, it's clear which remotes the
content gets dropped from.
Note that move --to=here will report an error if a non-reachable remote
contains the file, even if the local repository also contains the file. I
think that's reasonable; the user may be intending to move all other copies
of the file from remotes.
OTOH, if a copy of the file is believed to be present in some repository
that is not a configured remote, move --to=here does not report an error.
So a little bit inconsistent, but erroring in this case feels wrong.
copy --to=here came along for free, but it's basically the same behavior as
git-annex get, and probably with not as good messages in edge cases
(especially on failure), so I've not documented it.
This commit was sponsored by Anthony DeRobertis on Patreon.
2017-05-31 20:57:27 +00:00
|
|
|
|
|
|
|
{- Moves (or copies) the content of an annexed file from reachable remotes
|
|
|
|
- to the current repository.
|
|
|
|
-
|
2018-04-13 18:06:25 +00:00
|
|
|
- When moving, the content is removed from all the reachable remotes that
|
|
|
|
- it can safely be removed from. -}
|
2020-09-14 20:49:33 +00:00
|
|
|
toHereStart :: RemoveWhen -> AssociatedFile -> Key -> ActionItem -> SeekInput -> CommandStart
|
|
|
|
toHereStart removewhen afile key ai si =
|
2020-07-24 16:56:02 +00:00
|
|
|
startingNoMessage (OnlyActionOn key ai) $ do
|
2023-11-30 19:11:57 +00:00
|
|
|
rs <- Remote.keyPossibilities (Remote.IncludeIgnored False) key
|
move --to=here
* move --to=here moves from all reachable remotes to the local repository.
The output of move --from remote is changed slightly, when the remote and
local both have the content. It used to say:
move foo ok
Now:
move foo (from theremote...) ok
That was done so that, when move --to=here is used and the content is
locally present and also in several remotes, it's clear which remotes the
content gets dropped from.
Note that move --to=here will report an error if a non-reachable remote
contains the file, even if the local repository also contains the file. I
think that's reasonable; the user may be intending to move all other copies
of the file from remotes.
OTOH, if a copy of the file is believed to be present in some repository
that is not a configured remote, move --to=here does not report an error.
So a little bit inconsistent, but erroring in this case feels wrong.
copy --to=here came along for free, but it's basically the same behavior as
git-annex get, and probably with not as good messages in edge cases
(especially on failure), so I've not documented it.
This commit was sponsored by Anthony DeRobertis on Patreon.
2017-05-31 20:57:27 +00:00
|
|
|
forM_ rs $ \r ->
|
make CommandStart return a StartMessage
The goal is to be able to run CommandStart in the main thread when -J is
used, rather than unncessarily passing it off to a worker thread, which
incurs overhead that is signficant when the CommandStart is going to
quickly decide to stop.
To do that, the message it displays needs to be displayed in the worker
thread, after the CommandStart has run.
Also, the change will mean that CommandStart will no longer necessarily
run with the same Annex state as CommandPerform. While its docs already
said it should avoid modifying Annex state, I audited all the
CommandStart code as part of the conversion. (Note that CommandSeek
already sometimes runs with a different Annex state, and that has not been
a source of any problems, so I am not too worried that this change will
lead to breakage going forward.)
The only modification of Annex state I found was it calling
allowMessages in some Commands that default to noMessages. Dealt with
that by adding a startCustomOutput and a startingUsualMessages.
This lets a command start with noMessages and then select the output it
wants for each CommandStart.
One bit of breakage: onlyActionOn has been removed from commands that used it.
The plan is that, since a StartMessage contains an ActionItem,
when a Key can be extracted from that, the parallel job runner can
run onlyActionOn' automatically. Then commands won't need to worry about
this detail. Future work.
Otherwise, this was a fairly straightforward process of making each
CommandStart compile again. Hopefully other behavior changes were mostly
avoided.
In a few cases, a command had a CommandStart that called a CommandPerform
that then called showStart multiple times. I have collapsed those
down to a single start action. The main command to perhaps suffer from it
is Command.Direct, which used to show a start for each file, and no
longer does.
Another minor behavior change is that some commands used showStart
before, but had an associated file and a Key available, so were changed
to ShowStart with an ActionItemAssociatedFile. That will not change the
normal output or behavior, but --json output will now include the key.
This should not break it for anyone using a real json parser.
2019-06-06 19:42:30 +00:00
|
|
|
includeCommandAction $
|
2020-09-14 20:49:33 +00:00
|
|
|
starting (describeMoveAction removewhen) ai si $
|
make CommandStart return a StartMessage
The goal is to be able to run CommandStart in the main thread when -J is
used, rather than unncessarily passing it off to a worker thread, which
incurs overhead that is signficant when the CommandStart is going to
quickly decide to stop.
To do that, the message it displays needs to be displayed in the worker
thread, after the CommandStart has run.
Also, the change will mean that CommandStart will no longer necessarily
run with the same Annex state as CommandPerform. While its docs already
said it should avoid modifying Annex state, I audited all the
CommandStart code as part of the conversion. (Note that CommandSeek
already sometimes runs with a different Annex state, and that has not been
a source of any problems, so I am not too worried that this change will
lead to breakage going forward.)
The only modification of Annex state I found was it calling
allowMessages in some Commands that default to noMessages. Dealt with
that by adding a startCustomOutput and a startingUsualMessages.
This lets a command start with noMessages and then select the output it
wants for each CommandStart.
One bit of breakage: onlyActionOn has been removed from commands that used it.
The plan is that, since a StartMessage contains an ActionItem,
when a Key can be extracted from that, the parallel job runner can
run onlyActionOn' automatically. Then commands won't need to worry about
this detail. Future work.
Otherwise, this was a fairly straightforward process of making each
CommandStart compile again. Hopefully other behavior changes were mostly
avoided.
In a few cases, a command had a CommandStart that called a CommandPerform
that then called showStart multiple times. I have collapsed those
down to a single start action. The main command to perhaps suffer from it
is Command.Direct, which used to show a start for each file, and no
longer does.
Another minor behavior change is that some commands used showStart
before, but had an associated file and a Key available, so were changed
to ShowStart with an ActionItemAssociatedFile. That will not change the
normal output or behavior, but --json output will now include the key.
This should not break it for anyone using a real json parser.
2019-06-06 19:42:30 +00:00
|
|
|
fromPerform r removewhen key afile
|
finish CommandStart transition
The hoped for optimisation of CommandStart with -J did not materialize.
In fact, not runnign CommandStart in parallel is slower than -J3.
So, CommandStart are still run in parallel.
(The actual bad performance I've been seeing with -J in my big repo
has to do with building the remoteList.)
But, this is still progress toward making -J faster, because it gets rid
of the onlyActionOn roadblock in the way of making CommandCleanup jobs
run separate from CommandPerform jobs.
Added OnlyActionOn constructor for ActionItem which fixes the
onlyActionOn breakage in the last commit.
Made CustomOutput include an ActionItem, so even things using it can
specify OnlyActionOn.
In Command.Move and Command.Sync, there were CommandStarts that used
includeCommandAction, so output messages, which is no longer allowed.
Fixed by using startingCustomOutput, but that's still not quite right,
since it prevents message display for the includeCommandAction run
inside it too.
2019-06-12 13:23:26 +00:00
|
|
|
next $ return True
|
2018-04-13 18:06:25 +00:00
|
|
|
|
2023-01-20 15:10:38 +00:00
|
|
|
fromToStart :: RemoveWhen -> AssociatedFile -> Key -> ActionItem -> SeekInput -> Remote -> Remote -> CommandStart
|
2023-11-17 21:10:24 +00:00
|
|
|
fromToStart removewhen afile key ai si src dest =
|
|
|
|
stopUnless somethingtodo $ do
|
|
|
|
u <- getUUID
|
|
|
|
if u == Remote.uuid src
|
|
|
|
then toStart removewhen afile key ai si dest
|
|
|
|
else if u == Remote.uuid dest
|
|
|
|
then fromStart removewhen afile key ai si src
|
|
|
|
else stopUnless (fromOk src key) $
|
|
|
|
starting (describeMoveAction removewhen) (OnlyActionOn key ai) si $
|
|
|
|
fromToPerform src dest removewhen key afile
|
|
|
|
where
|
2023-11-17 21:27:58 +00:00
|
|
|
somethingtodo
|
|
|
|
| Remote.uuid src == Remote.uuid dest = return False
|
|
|
|
| otherwise = do
|
|
|
|
fast <- Annex.getRead Annex.fast
|
|
|
|
if fast && removewhen == RemoveNever
|
|
|
|
then not <$> expectedPresent dest key
|
|
|
|
else return True
|
2023-01-20 15:10:38 +00:00
|
|
|
|
copy/move --from-anywhere --to remote
Implementation was simple because it's equivilant to
--from=foo --to remote for each other remote, followed by
--to remote when there's a local copy.
(Or, in the edge case of --from-anywhere --to=here,
it's the same as --to=here.)
Note that, when the local repo does not have a copy,
fromToPerform gets it from a remote, sends it to the destination,
and drops the local copy. Another call to that for a second remote
will notice that the dest now has a copy, and simply drop from the
second remote, avoiding a second transfer.
Also note that, when numcopies doesn't allow dropping it from
everywhere, it will drop it from the cheapest remotes first
(maybe not ideal) up to more expensive remotes, and finally from the local
repo. So the local repo will generally end up holding a copy. Maybe not
ideal in all cases either, but it seems no worse to do that than to end up
with a copy undropped from a remote.
And I'm not entirely happy with the output, eg:
copy bigfile (from r3...) ok
copy bigfile ok
That makes sense if you think of the second line as being
the same as what is output by `git-annex copy bigfile --to bar`,
but it's less clear in this context. Maybe add "(from here...)"?
Also the --json output doesn't have a machine-readable field for
the "from" uuid, and maybe it should?
Sponsored-by: Dartmouth College's DANDI project
2023-11-30 20:32:32 +00:00
|
|
|
fromAnywhereToStart :: RemoveWhen -> AssociatedFile -> Key -> ActionItem -> SeekInput -> Remote -> CommandStart
|
|
|
|
fromAnywhereToStart removewhen afile key ai si dest =
|
|
|
|
stopUnless somethingtodo $ do
|
|
|
|
u <- getUUID
|
|
|
|
if u == Remote.uuid dest
|
|
|
|
then toHereStart removewhen afile key ai si
|
|
|
|
else startingNoMessage (OnlyActionOn key ai) $ do
|
|
|
|
rs <- filter (/= dest)
|
|
|
|
<$> Remote.keyPossibilities (Remote.IncludeIgnored False) key
|
|
|
|
forM_ rs $ \r ->
|
|
|
|
includeCommandAction $
|
|
|
|
starting (describeMoveAction removewhen) ai si $
|
|
|
|
fromToPerform r dest removewhen key afile
|
|
|
|
whenM (inAnnex key) $
|
|
|
|
void $ includeCommandAction $
|
|
|
|
toStart removewhen afile key ai si dest
|
|
|
|
next $ return True
|
|
|
|
where
|
|
|
|
somethingtodo = do
|
|
|
|
fast <- Annex.getRead Annex.fast
|
|
|
|
if fast && removewhen == RemoveNever
|
|
|
|
then not <$> expectedPresent dest key
|
|
|
|
else return True
|
|
|
|
|
2023-01-23 17:16:14 +00:00
|
|
|
{- When there is a local copy, transfer it to the dest, and drop from the src.
|
fully working move --from --to (not release quality)
When the destination already has a copy, it behaves the same as
drop --from really, but display it as a move and implement it
reusing the factored out code from fromPerform.
(Note that willDropMakeItWorse never returns DropAllowed in that
situation, because it's told that dest has a copy. So numcopies is
always checked.)
And when only the source and not the local repo or destination have a
copy, do the full copy from source to local, then copy from local to
dest, then drop from local, then drop from source dance.
This is complicated by fromPerform being hardcoded to assume there is a
local copy, but the local copy has already been dropped. That's why
it uses cleanupfromsrc RemoveNever to avoid the code that makes that
assumption, and finishes with a call to dropfromsrc.
And, since the location log has not yet been updated, checking numcopies
was not working, until I added UnVerifiedRemote dest to the list of
things to check.
This is not yet quite mergeable though. There are two things in the
comment above fromToPerform that are not implemented yet: Checking the
location log before dropping the local copy, and locking the temporary
local copy for drop.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 19:36:06 +00:00
|
|
|
-
|
|
|
|
- When the dest has a copy, drop it from the src.
|
2023-01-23 17:16:14 +00:00
|
|
|
-
|
2023-01-20 15:10:38 +00:00
|
|
|
- Otherwise, download a copy from the dest, populating the local annex
|
|
|
|
- copy, but not updating location logs. Then transfer that to the dest,
|
|
|
|
- drop the local copy, and finally drop from the src.
|
|
|
|
-
|
|
|
|
- Using a regular download of the local copy, rather than download to
|
2023-03-14 02:39:16 +00:00
|
|
|
- some other file makes resuming an interrupted download work as usual,
|
2023-01-23 17:16:14 +00:00
|
|
|
- and simplifies implementation. It does mean that, if `git-annex get` of
|
finishing up move --from --to
Lock the local content for drop after getting it from src, to prevent another
process from using the local content as a copy and dropping it from src,
which would prevent dropping the local content after sending it to dest.
Support resuming an interrupted move that downloaded the content from
src, leaving the local content populated. In this case, the location log
has not been updated to say the content is present locally, so we can
assume that it's resuming and go ahead and drop the local content after
sending it to dest.
Note that if a `git-annex get` is being ran at the same time as a
`git-annex move --from --to`, it may get a file just before the move
processes it. So the location log has not been updated yet, and the move
thinks it's resuming. Resulting in local copy being dropped after it's
sent to the dest. This race is something we'll just have to live with,
it seems.
I also gave up on the idea of checking if the location log had been updated
by a `git-annex get` that is ran at the same time. That wouldn't work, because
the location log is precached in the seek stage, so reading it again after
sending the content to dest would not notice changes made to it, unless the cache
were invalidated, which would slow it down a lot. That idea anyway was subject
to races where it would not detect the concurrent `git-annex get`.
So concurrent `git-annex get` will have results that may be surprising.
To make that less surprising, updated the documentation of this feature to
be explicit that it downloads content to the local repository
temporarily.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 21:07:21 +00:00
|
|
|
- the same content is being run at the same time as this move, the content
|
|
|
|
- may end up locally present, or not. This is similar to the behavior
|
|
|
|
- when running `git-annex move --to` concurrently with git-annex get.
|
2023-01-20 15:10:38 +00:00
|
|
|
-}
|
|
|
|
fromToPerform :: Remote -> Remote -> RemoveWhen -> Key -> AssociatedFile -> CommandPerform
|
finishing up move --from --to
Lock the local content for drop after getting it from src, to prevent another
process from using the local content as a copy and dropping it from src,
which would prevent dropping the local content after sending it to dest.
Support resuming an interrupted move that downloaded the content from
src, leaving the local content populated. In this case, the location log
has not been updated to say the content is present locally, so we can
assume that it's resuming and go ahead and drop the local content after
sending it to dest.
Note that if a `git-annex get` is being ran at the same time as a
`git-annex move --from --to`, it may get a file just before the move
processes it. So the location log has not been updated yet, and the move
thinks it's resuming. Resulting in local copy being dropped after it's
sent to the dest. This race is something we'll just have to live with,
it seems.
I also gave up on the idea of checking if the location log had been updated
by a `git-annex get` that is ran at the same time. That wouldn't work, because
the location log is precached in the seek stage, so reading it again after
sending the content to dest would not notice changes made to it, unless the cache
were invalidated, which would slow it down a lot. That idea anyway was subject
to races where it would not detect the concurrent `git-annex get`.
So concurrent `git-annex get` will have results that may be surprising.
To make that less surprising, updated the documentation of this feature to
be explicit that it downloads content to the local repository
temporarily.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 21:07:21 +00:00
|
|
|
fromToPerform src dest removewhen key afile = do
|
|
|
|
hereuuid <- getUUID
|
|
|
|
loggedpresent <- any (== hereuuid)
|
|
|
|
<$> loggedLocations key
|
|
|
|
ispresent <- inAnnex key
|
|
|
|
go ispresent loggedpresent
|
2023-01-20 15:10:38 +00:00
|
|
|
where
|
finishing up move --from --to
Lock the local content for drop after getting it from src, to prevent another
process from using the local content as a copy and dropping it from src,
which would prevent dropping the local content after sending it to dest.
Support resuming an interrupted move that downloaded the content from
src, leaving the local content populated. In this case, the location log
has not been updated to say the content is present locally, so we can
assume that it's resuming and go ahead and drop the local content after
sending it to dest.
Note that if a `git-annex get` is being ran at the same time as a
`git-annex move --from --to`, it may get a file just before the move
processes it. So the location log has not been updated yet, and the move
thinks it's resuming. Resulting in local copy being dropped after it's
sent to the dest. This race is something we'll just have to live with,
it seems.
I also gave up on the idea of checking if the location log had been updated
by a `git-annex get` that is ran at the same time. That wouldn't work, because
the location log is precached in the seek stage, so reading it again after
sending the content to dest would not notice changes made to it, unless the cache
were invalidated, which would slow it down a lot. That idea anyway was subject
to races where it would not detect the concurrent `git-annex get`.
So concurrent `git-annex get` will have results that may be surprising.
To make that less surprising, updated the documentation of this feature to
be explicit that it downloads content to the local repository
temporarily.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 21:07:21 +00:00
|
|
|
-- The content is present, and is logged as present, so it
|
|
|
|
-- can be sent to dest and dropped from src.
|
|
|
|
--
|
|
|
|
-- When resuming an interrupted move --from --to, where the content
|
|
|
|
-- was not present but got downloaded from src, it will not be
|
|
|
|
-- logged present, and so this won't be used. Instead, the local
|
|
|
|
-- content will get dropped after being copied to dest.
|
|
|
|
go True True = do
|
fully working move --from --to (not release quality)
When the destination already has a copy, it behaves the same as
drop --from really, but display it as a move and implement it
reusing the factored out code from fromPerform.
(Note that willDropMakeItWorse never returns DropAllowed in that
situation, because it's told that dest has a copy. So numcopies is
always checked.)
And when only the source and not the local repo or destination have a
copy, do the full copy from source to local, then copy from local to
dest, then drop from local, then drop from source dance.
This is complicated by fromPerform being hardcoded to assume there is a
local copy, but the local copy has already been dropped. That's why
it uses cleanupfromsrc RemoveNever to avoid the code that makes that
assumption, and finishes with a call to dropfromsrc.
And, since the location log has not yet been updated, checking numcopies
was not working, until I added UnVerifiedRemote dest to the list of
things to check.
This is not yet quite mergeable though. There are two things in the
comment above fromToPerform that are not implemented yet: Checking the
location log before dropping the local copy, and locking the temporary
local copy for drop.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 19:36:06 +00:00
|
|
|
haskey <- Remote.hasKey dest key
|
|
|
|
-- Prepare to drop from src later. Doing this first
|
|
|
|
-- makes "from src" be shown consistently before
|
|
|
|
-- "to dest"
|
|
|
|
dropsrc <- fromsrc True
|
|
|
|
combinecleanups
|
|
|
|
-- Send to dest, preserve local copy.
|
finishing up move --from --to
Lock the local content for drop after getting it from src, to prevent another
process from using the local content as a copy and dropping it from src,
which would prevent dropping the local content after sending it to dest.
Support resuming an interrupted move that downloaded the content from
src, leaving the local content populated. In this case, the location log
has not been updated to say the content is present locally, so we can
assume that it's resuming and go ahead and drop the local content after
sending it to dest.
Note that if a `git-annex get` is being ran at the same time as a
`git-annex move --from --to`, it may get a file just before the move
processes it. So the location log has not been updated yet, and the move
thinks it's resuming. Resulting in local copy being dropped after it's
sent to the dest. This race is something we'll just have to live with,
it seems.
I also gave up on the idea of checking if the location log had been updated
by a `git-annex get` that is ran at the same time. That wouldn't work, because
the location log is precached in the seek stage, so reading it again after
sending the content to dest would not notice changes made to it, unless the cache
were invalidated, which would slow it down a lot. That idea anyway was subject
to races where it would not detect the concurrent `git-annex get`.
So concurrent `git-annex get` will have results that may be surprising.
To make that less surprising, updated the documentation of this feature to
be explicit that it downloads content to the local repository
temporarily.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 21:07:21 +00:00
|
|
|
(todest Nothing RemoveNever haskey)
|
fully working move --from --to (not release quality)
When the destination already has a copy, it behaves the same as
drop --from really, but display it as a move and implement it
reusing the factored out code from fromPerform.
(Note that willDropMakeItWorse never returns DropAllowed in that
situation, because it's told that dest has a copy. So numcopies is
always checked.)
And when only the source and not the local repo or destination have a
copy, do the full copy from source to local, then copy from local to
dest, then drop from local, then drop from source dance.
This is complicated by fromPerform being hardcoded to assume there is a
local copy, but the local copy has already been dropped. That's why
it uses cleanupfromsrc RemoveNever to avoid the code that makes that
assumption, and finishes with a call to dropfromsrc.
And, since the location log has not yet been updated, checking numcopies
was not working, until I added UnVerifiedRemote dest to the list of
things to check.
This is not yet quite mergeable though. There are two things in the
comment above fromToPerform that are not implemented yet: Checking the
location log before dropping the local copy, and locking the temporary
local copy for drop.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 19:36:06 +00:00
|
|
|
(\senttodest -> if senttodest
|
|
|
|
then dropsrc removewhen
|
|
|
|
else stop
|
|
|
|
)
|
finishing up move --from --to
Lock the local content for drop after getting it from src, to prevent another
process from using the local content as a copy and dropping it from src,
which would prevent dropping the local content after sending it to dest.
Support resuming an interrupted move that downloaded the content from
src, leaving the local content populated. In this case, the location log
has not been updated to say the content is present locally, so we can
assume that it's resuming and go ahead and drop the local content after
sending it to dest.
Note that if a `git-annex get` is being ran at the same time as a
`git-annex move --from --to`, it may get a file just before the move
processes it. So the location log has not been updated yet, and the move
thinks it's resuming. Resulting in local copy being dropped after it's
sent to the dest. This race is something we'll just have to live with,
it seems.
I also gave up on the idea of checking if the location log had been updated
by a `git-annex get` that is ran at the same time. That wouldn't work, because
the location log is precached in the seek stage, so reading it again after
sending the content to dest would not notice changes made to it, unless the cache
were invalidated, which would slow it down a lot. That idea anyway was subject
to races where it would not detect the concurrent `git-annex get`.
So concurrent `git-annex get` will have results that may be surprising.
To make that less surprising, updated the documentation of this feature to
be explicit that it downloads content to the local repository
temporarily.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 21:07:21 +00:00
|
|
|
go ispresent _loggedpresent = do
|
2023-01-23 17:16:14 +00:00
|
|
|
haskey <- Remote.hasKey dest key
|
fully working move --from --to (not release quality)
When the destination already has a copy, it behaves the same as
drop --from really, but display it as a move and implement it
reusing the factored out code from fromPerform.
(Note that willDropMakeItWorse never returns DropAllowed in that
situation, because it's told that dest has a copy. So numcopies is
always checked.)
And when only the source and not the local repo or destination have a
copy, do the full copy from source to local, then copy from local to
dest, then drop from local, then drop from source dance.
This is complicated by fromPerform being hardcoded to assume there is a
local copy, but the local copy has already been dropped. That's why
it uses cleanupfromsrc RemoveNever to avoid the code that makes that
assumption, and finishes with a call to dropfromsrc.
And, since the location log has not yet been updated, checking numcopies
was not working, until I added UnVerifiedRemote dest to the list of
things to check.
This is not yet quite mergeable though. There are two things in the
comment above fromToPerform that are not implemented yet: Checking the
location log before dropping the local copy, and locking the temporary
local copy for drop.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 19:36:06 +00:00
|
|
|
case haskey of
|
|
|
|
Left err -> do
|
2023-04-10 21:03:41 +00:00
|
|
|
showNote (UnquotedString err)
|
fully working move --from --to (not release quality)
When the destination already has a copy, it behaves the same as
drop --from really, but display it as a move and implement it
reusing the factored out code from fromPerform.
(Note that willDropMakeItWorse never returns DropAllowed in that
situation, because it's told that dest has a copy. So numcopies is
always checked.)
And when only the source and not the local repo or destination have a
copy, do the full copy from source to local, then copy from local to
dest, then drop from local, then drop from source dance.
This is complicated by fromPerform being hardcoded to assume there is a
local copy, but the local copy has already been dropped. That's why
it uses cleanupfromsrc RemoveNever to avoid the code that makes that
assumption, and finishes with a call to dropfromsrc.
And, since the location log has not yet been updated, checking numcopies
was not working, until I added UnVerifiedRemote dest to the list of
things to check.
This is not yet quite mergeable though. There are two things in the
comment above fromToPerform that are not implemented yet: Checking the
location log before dropping the local copy, and locking the temporary
local copy for drop.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 19:36:06 +00:00
|
|
|
stop
|
|
|
|
Right True -> do
|
2023-04-10 21:03:41 +00:00
|
|
|
showAction $ UnquotedString $
|
|
|
|
"from " ++ Remote.name src
|
|
|
|
showAction $ UnquotedString $
|
|
|
|
"to " ++ Remote.name dest
|
2023-03-13 18:50:28 +00:00
|
|
|
-- The log may not indicate dest's copy
|
|
|
|
-- yet, so make sure it does.
|
|
|
|
logChange key (Remote.uuid dest) InfoPresent
|
fully working move --from --to (not release quality)
When the destination already has a copy, it behaves the same as
drop --from really, but display it as a move and implement it
reusing the factored out code from fromPerform.
(Note that willDropMakeItWorse never returns DropAllowed in that
situation, because it's told that dest has a copy. So numcopies is
always checked.)
And when only the source and not the local repo or destination have a
copy, do the full copy from source to local, then copy from local to
dest, then drop from local, then drop from source dance.
This is complicated by fromPerform being hardcoded to assume there is a
local copy, but the local copy has already been dropped. That's why
it uses cleanupfromsrc RemoveNever to avoid the code that makes that
assumption, and finishes with a call to dropfromsrc.
And, since the location log has not yet been updated, checking numcopies
was not working, until I added UnVerifiedRemote dest to the list of
things to check.
This is not yet quite mergeable though. There are two things in the
comment above fromToPerform that are not implemented yet: Checking the
location log before dropping the local copy, and locking the temporary
local copy for drop.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 19:36:06 +00:00
|
|
|
-- Drop from src, checking copies including
|
|
|
|
-- the one already in dest.
|
|
|
|
dropfromsrc id
|
|
|
|
Right False -> do
|
|
|
|
-- Get local copy from src, defer dropping
|
finishing up move --from --to
Lock the local content for drop after getting it from src, to prevent another
process from using the local content as a copy and dropping it from src,
which would prevent dropping the local content after sending it to dest.
Support resuming an interrupted move that downloaded the content from
src, leaving the local content populated. In this case, the location log
has not been updated to say the content is present locally, so we can
assume that it's resuming and go ahead and drop the local content after
sending it to dest.
Note that if a `git-annex get` is being ran at the same time as a
`git-annex move --from --to`, it may get a file just before the move
processes it. So the location log has not been updated yet, and the move
thinks it's resuming. Resulting in local copy being dropped after it's
sent to the dest. This race is something we'll just have to live with,
it seems.
I also gave up on the idea of checking if the location log had been updated
by a `git-annex get` that is ran at the same time. That wouldn't work, because
the location log is precached in the seek stage, so reading it again after
sending the content to dest would not notice changes made to it, unless the cache
were invalidated, which would slow it down a lot. That idea anyway was subject
to races where it would not detect the concurrent `git-annex get`.
So concurrent `git-annex get` will have results that may be surprising.
To make that less surprising, updated the documentation of this feature to
be explicit that it downloads content to the local repository
temporarily.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 21:07:21 +00:00
|
|
|
-- from src until later. Note that fromsrc
|
|
|
|
-- does not update the location log.
|
|
|
|
cleanupfromsrc <- if ispresent
|
|
|
|
then return $ const $ next (return True)
|
|
|
|
else fromsrc False
|
|
|
|
-- Lock the local copy for removal early,
|
|
|
|
-- to avoid other processes relying on it
|
|
|
|
-- as a copy, and removing other copies
|
|
|
|
-- (such as the one in src), that prevents
|
|
|
|
-- dropping the local copy later.
|
|
|
|
lockContentForRemoval key stop $ \contentlock ->
|
|
|
|
combinecleanups
|
|
|
|
-- Send to dest and remove local copy.
|
|
|
|
(todest (Just contentlock) RemoveSafe haskey)
|
|
|
|
(\senttodest ->
|
|
|
|
-- Drop from src, checking
|
|
|
|
-- copies including dest.
|
|
|
|
combinecleanups
|
|
|
|
(cleanupfromsrc RemoveNever)
|
|
|
|
(\_ -> if senttodest
|
|
|
|
then dropfromsrc (\l -> UnVerifiedRemote dest : l)
|
|
|
|
else stop
|
|
|
|
)
|
|
|
|
)
|
fully working move --from --to (not release quality)
When the destination already has a copy, it behaves the same as
drop --from really, but display it as a move and implement it
reusing the factored out code from fromPerform.
(Note that willDropMakeItWorse never returns DropAllowed in that
situation, because it's told that dest has a copy. So numcopies is
always checked.)
And when only the source and not the local repo or destination have a
copy, do the full copy from source to local, then copy from local to
dest, then drop from local, then drop from source dance.
This is complicated by fromPerform being hardcoded to assume there is a
local copy, but the local copy has already been dropped. That's why
it uses cleanupfromsrc RemoveNever to avoid the code that makes that
assumption, and finishes with a call to dropfromsrc.
And, since the location log has not yet been updated, checking numcopies
was not working, until I added UnVerifiedRemote dest to the list of
things to check.
This is not yet quite mergeable though. There are two things in the
comment above fromToPerform that are not implemented yet: Checking the
location log before dropping the local copy, and locking the temporary
local copy for drop.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 19:36:06 +00:00
|
|
|
|
|
|
|
fromsrc present = fromPerform' present False src key afile
|
|
|
|
|
finishing up move --from --to
Lock the local content for drop after getting it from src, to prevent another
process from using the local content as a copy and dropping it from src,
which would prevent dropping the local content after sending it to dest.
Support resuming an interrupted move that downloaded the content from
src, leaving the local content populated. In this case, the location log
has not been updated to say the content is present locally, so we can
assume that it's resuming and go ahead and drop the local content after
sending it to dest.
Note that if a `git-annex get` is being ran at the same time as a
`git-annex move --from --to`, it may get a file just before the move
processes it. So the location log has not been updated yet, and the move
thinks it's resuming. Resulting in local copy being dropped after it's
sent to the dest. This race is something we'll just have to live with,
it seems.
I also gave up on the idea of checking if the location log had been updated
by a `git-annex get` that is ran at the same time. That wouldn't work, because
the location log is precached in the seek stage, so reading it again after
sending the content to dest would not notice changes made to it, unless the cache
were invalidated, which would slow it down a lot. That idea anyway was subject
to races where it would not detect the concurrent `git-annex get`.
So concurrent `git-annex get` will have results that may be surprising.
To make that less surprising, updated the documentation of this feature to
be explicit that it downloads content to the local repository
temporarily.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 21:07:21 +00:00
|
|
|
todest mcontentlock removewhen' = toPerform' mcontentlock dest removewhen' key afile False
|
fully working move --from --to (not release quality)
When the destination already has a copy, it behaves the same as
drop --from really, but display it as a move and implement it
reusing the factored out code from fromPerform.
(Note that willDropMakeItWorse never returns DropAllowed in that
situation, because it's told that dest has a copy. So numcopies is
always checked.)
And when only the source and not the local repo or destination have a
copy, do the full copy from source to local, then copy from local to
dest, then drop from local, then drop from source dance.
This is complicated by fromPerform being hardcoded to assume there is a
local copy, but the local copy has already been dropped. That's why
it uses cleanupfromsrc RemoveNever to avoid the code that makes that
assumption, and finishes with a call to dropfromsrc.
And, since the location log has not yet been updated, checking numcopies
was not working, until I added UnVerifiedRemote dest to the list of
things to check.
This is not yet quite mergeable though. There are two things in the
comment above fromToPerform that are not implemented yet: Checking the
location log before dropping the local copy, and locking the temporary
local copy for drop.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 19:36:06 +00:00
|
|
|
|
2023-01-23 21:53:57 +00:00
|
|
|
dropfromsrc adjusttocheck = case removewhen of
|
|
|
|
RemoveSafe -> logMove (Remote.uuid src) (Remote.uuid dest) True key $ \deststartedwithcopy ->
|
fully working move --from --to (not release quality)
When the destination already has a copy, it behaves the same as
drop --from really, but display it as a move and implement it
reusing the factored out code from fromPerform.
(Note that willDropMakeItWorse never returns DropAllowed in that
situation, because it's told that dest has a copy. So numcopies is
always checked.)
And when only the source and not the local repo or destination have a
copy, do the full copy from source to local, then copy from local to
dest, then drop from local, then drop from source dance.
This is complicated by fromPerform being hardcoded to assume there is a
local copy, but the local copy has already been dropped. That's why
it uses cleanupfromsrc RemoveNever to avoid the code that makes that
assumption, and finishes with a call to dropfromsrc.
And, since the location log has not yet been updated, checking numcopies
was not working, until I added UnVerifiedRemote dest to the list of
things to check.
This is not yet quite mergeable though. There are two things in the
comment above fromToPerform that are not implemented yet: Checking the
location log before dropping the local copy, and locking the temporary
local copy for drop.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 19:36:06 +00:00
|
|
|
fromDrop src (Remote.uuid dest) deststartedwithcopy key afile adjusttocheck
|
2023-01-23 21:53:57 +00:00
|
|
|
RemoveNever -> next (return True)
|
fully working move --from --to (not release quality)
When the destination already has a copy, it behaves the same as
drop --from really, but display it as a move and implement it
reusing the factored out code from fromPerform.
(Note that willDropMakeItWorse never returns DropAllowed in that
situation, because it's told that dest has a copy. So numcopies is
always checked.)
And when only the source and not the local repo or destination have a
copy, do the full copy from source to local, then copy from local to
dest, then drop from local, then drop from source dance.
This is complicated by fromPerform being hardcoded to assume there is a
local copy, but the local copy has already been dropped. That's why
it uses cleanupfromsrc RemoveNever to avoid the code that makes that
assumption, and finishes with a call to dropfromsrc.
And, since the location log has not yet been updated, checking numcopies
was not working, until I added UnVerifiedRemote dest to the list of
things to check.
This is not yet quite mergeable though. There are two things in the
comment above fromToPerform that are not implemented yet: Checking the
location log before dropping the local copy, and locking the temporary
local copy for drop.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 19:36:06 +00:00
|
|
|
|
|
|
|
combinecleanups a b = a >>= \case
|
|
|
|
Just cleanupa -> b True >>= \case
|
|
|
|
Just cleanupb -> return $ Just $ do
|
|
|
|
oka <- cleanupa
|
|
|
|
okb <- cleanupb
|
|
|
|
return (oka && okb)
|
|
|
|
Nothing -> return (Just cleanupa)
|
|
|
|
Nothing -> b False >>= \case
|
|
|
|
Just cleanupb -> return $ Just $ do
|
|
|
|
void cleanupb
|
|
|
|
return False
|
2023-01-23 17:16:14 +00:00
|
|
|
Nothing -> return Nothing
|
|
|
|
|
2018-04-13 18:06:25 +00:00
|
|
|
{- The goal of this command is to allow the user maximum freedom to move
|
|
|
|
- files as they like, while avoiding making bad situations any worse
|
|
|
|
- than they already were.
|
|
|
|
-
|
|
|
|
- When the destination repository already had a copy of a file
|
|
|
|
- before the move operation began, dropping it from the source
|
|
|
|
- repository reduces the number of copies, and should fail if
|
|
|
|
- that would violate numcopies settings.
|
|
|
|
-
|
fully working move --from --to (not release quality)
When the destination already has a copy, it behaves the same as
drop --from really, but display it as a move and implement it
reusing the factored out code from fromPerform.
(Note that willDropMakeItWorse never returns DropAllowed in that
situation, because it's told that dest has a copy. So numcopies is
always checked.)
And when only the source and not the local repo or destination have a
copy, do the full copy from source to local, then copy from local to
dest, then drop from local, then drop from source dance.
This is complicated by fromPerform being hardcoded to assume there is a
local copy, but the local copy has already been dropped. That's why
it uses cleanupfromsrc RemoveNever to avoid the code that makes that
assumption, and finishes with a call to dropfromsrc.
And, since the location log has not yet been updated, checking numcopies
was not working, until I added UnVerifiedRemote dest to the list of
things to check.
This is not yet quite mergeable though. There are two things in the
comment above fromToPerform that are not implemented yet: Checking the
location log before dropping the local copy, and locking the temporary
local copy for drop.
Sponsored-by: Dartmouth College's DANDI project
2023-01-23 19:36:06 +00:00
|
|
|
- On the other hand, when the destination repository did not start
|
|
|
|
- with a copy of a file, it can be dropped from the source without
|
|
|
|
- making numcopies worse, so the move is allowed even if numcopies
|
2024-06-23 16:00:50 +00:00
|
|
|
- is not met. (However, when the source is a cluster, dropping from it
|
|
|
|
- drops from all nodes, and so numcopies must be checked.)
|
2018-04-13 18:06:25 +00:00
|
|
|
-
|
|
|
|
- Similarly, a file can move from an untrusted repository to another
|
|
|
|
- untrusted repository, even if that is the only copy of the file.
|
|
|
|
-
|
|
|
|
- But, moving a file from a repository with higher trust to an untrusted
|
|
|
|
- repository must still check that there are enough other copies to be
|
|
|
|
- safe.
|
|
|
|
-
|
|
|
|
- Also, required content settings should not be violated.
|
|
|
|
-
|
|
|
|
- This function checks all that. It needs to know if the destination
|
|
|
|
- repository already had a copy of the file before the move began.
|
|
|
|
-}
|
2020-10-21 14:31:56 +00:00
|
|
|
willDropMakeItWorse :: UUID -> UUID -> DestStartedWithCopy -> Key -> AssociatedFile -> Annex DropCheck
|
2022-06-09 19:26:25 +00:00
|
|
|
willDropMakeItWorse srcuuid destuuid (DestStartedWithCopy deststartedwithcopy _) key afile =
|
2021-05-25 14:57:06 +00:00
|
|
|
ifM (Command.Drop.checkRequiredContent (Command.Drop.PreferredContentChecked False) srcuuid key afile)
|
2024-06-23 16:00:50 +00:00
|
|
|
( if deststartedwithcopy || isClusterUUID srcuuid
|
2018-04-13 18:06:25 +00:00
|
|
|
then unlessforced DropCheckNumCopies
|
|
|
|
else ifM checktrustlevel
|
|
|
|
( return DropAllowed
|
|
|
|
, unlessforced DropCheckNumCopies
|
|
|
|
)
|
|
|
|
, unlessforced DropWorse
|
|
|
|
)
|
|
|
|
where
|
2022-06-28 19:28:14 +00:00
|
|
|
unlessforced r = ifM (Annex.getRead Annex.force)
|
2018-04-13 18:06:25 +00:00
|
|
|
( return DropAllowed
|
|
|
|
, return r
|
|
|
|
)
|
|
|
|
checktrustlevel = do
|
|
|
|
desttrust <- lookupTrust destuuid
|
|
|
|
srctrust <- lookupTrust srcuuid
|
2018-04-13 19:16:07 +00:00
|
|
|
return (desttrust > UnTrusted || desttrust >= srctrust)
|
2018-04-13 18:06:25 +00:00
|
|
|
|
|
|
|
data DropCheck = DropWorse | DropAllowed | DropCheckNumCopies
|
2020-10-21 14:31:56 +00:00
|
|
|
|
2022-06-09 19:26:25 +00:00
|
|
|
data DestStartedWithCopy = DestStartedWithCopy Bool (Annex ())
|
|
|
|
|
|
|
|
{- This should be called once the move has succeeded, or if it failed
|
|
|
|
- without doing anything. It should not be called if the move transferred
|
|
|
|
- the content but failed to drop due to eg a network error. In such a
|
|
|
|
- case, the move can be restarted later, so the move log should be
|
|
|
|
- preserved. -}
|
|
|
|
logMoveCleanup :: DestStartedWithCopy -> Annex ()
|
|
|
|
logMoveCleanup (DestStartedWithCopy _ a) = a
|
2020-10-21 14:31:56 +00:00
|
|
|
|
|
|
|
{- Runs an action that performs a move, and logs the move, allowing an
|
2022-06-09 19:26:25 +00:00
|
|
|
- failed or interrupted move to be re-done later.
|
2020-10-21 14:31:56 +00:00
|
|
|
-
|
|
|
|
- This deals with the situation where dest did not start with a copy,
|
|
|
|
- but the move downloaded it, and was then interrupted before dropping
|
|
|
|
- it from the source. Re-running the move would see dest has a
|
|
|
|
- copy, and so could refuse to allow the drop. By providing the logged
|
|
|
|
- DestStartedWithCopy, this avoids that annoyance.
|
|
|
|
-}
|
|
|
|
logMove :: UUID -> UUID -> Bool -> Key -> (DestStartedWithCopy -> Annex a) -> Annex a
|
2022-06-09 19:26:25 +00:00
|
|
|
logMove srcuuid destuuid deststartedwithcopy key a = go =<< setup
|
2020-10-21 14:31:56 +00:00
|
|
|
where
|
|
|
|
logline = L.fromStrict $ B8.unwords
|
|
|
|
[ fromUUID srcuuid
|
|
|
|
, fromUUID destuuid
|
|
|
|
, serializeKey' key
|
|
|
|
]
|
|
|
|
|
|
|
|
setup = do
|
|
|
|
logf <- fromRepo gitAnnexMoveLog
|
2022-08-11 20:57:44 +00:00
|
|
|
lckf <- fromRepo gitAnnexMoveLock
|
2020-10-21 14:31:56 +00:00
|
|
|
-- Only log when there was no copy.
|
|
|
|
unless deststartedwithcopy $
|
2022-08-11 20:57:44 +00:00
|
|
|
appendLogFile logf lckf logline
|
2022-10-07 17:19:17 +00:00
|
|
|
return (logf, lckf)
|
2020-10-21 14:31:56 +00:00
|
|
|
|
2022-10-07 17:19:17 +00:00
|
|
|
cleanup (logf, lckf) =
|
2020-10-21 14:31:56 +00:00
|
|
|
-- This buffers the log file content in memory.
|
|
|
|
-- The log file length is limited to the number of
|
|
|
|
-- concurrent jobs, times the number of times a move
|
|
|
|
-- (of different files) has been interrupted.
|
|
|
|
-- That could grow without bounds given enough time,
|
|
|
|
-- so the log is also truncated to the most recent
|
|
|
|
-- 100 items.
|
2022-10-07 17:19:17 +00:00
|
|
|
modifyLogFile logf lckf
|
2020-10-21 14:31:56 +00:00
|
|
|
(filter (/= logline) . reverse . take 100 . reverse)
|
|
|
|
|
2022-10-07 17:19:17 +00:00
|
|
|
go fs@(logf, lckf)
|
2020-10-21 14:31:56 +00:00
|
|
|
-- Only need to check log when there is a copy.
|
|
|
|
| deststartedwithcopy = do
|
2022-10-07 17:19:17 +00:00
|
|
|
wasnocopy <- checkLogFile logf lckf (== logline)
|
2020-10-21 14:31:56 +00:00
|
|
|
if wasnocopy
|
2022-10-07 17:19:17 +00:00
|
|
|
then go' fs False
|
|
|
|
else go' fs deststartedwithcopy
|
|
|
|
| otherwise = go' fs deststartedwithcopy
|
2020-10-21 14:31:56 +00:00
|
|
|
|
2022-10-07 17:19:17 +00:00
|
|
|
go' fs deststartedwithcopy' = a $
|
|
|
|
DestStartedWithCopy deststartedwithcopy' (cleanup fs)
|