2011-04-28 00:06:07 +00:00
|
|
|
{- A remote that is only accessible by rsync.
|
|
|
|
-
|
2020-01-14 16:35:08 +00:00
|
|
|
- Copyright 2011-2020 Joey Hess <id@joeyh.name>
|
2011-04-28 00:06:07 +00:00
|
|
|
-
|
2019-03-13 19:48:14 +00:00
|
|
|
- Licensed under the GNU AGPL version 3 or higher.
|
2011-04-28 00:06:07 +00:00
|
|
|
-}
|
|
|
|
|
2013-05-10 21:29:59 +00:00
|
|
|
{-# LANGUAGE CPP #-}
|
|
|
|
|
2013-09-08 18:54:28 +00:00
|
|
|
module Remote.Rsync (
|
|
|
|
remote,
|
2014-08-03 21:31:10 +00:00
|
|
|
store,
|
|
|
|
retrieve,
|
2013-09-08 18:54:28 +00:00
|
|
|
remove,
|
2014-08-06 17:45:19 +00:00
|
|
|
checkKey,
|
2013-09-08 18:54:28 +00:00
|
|
|
withRsyncScratchDir,
|
2020-01-20 19:13:49 +00:00
|
|
|
rsyncRemoteConfigs,
|
2013-09-08 18:54:28 +00:00
|
|
|
genRsyncOpts,
|
|
|
|
RsyncOpts
|
|
|
|
) where
|
2011-04-28 00:06:07 +00:00
|
|
|
|
2016-01-20 20:36:33 +00:00
|
|
|
import Annex.Common
|
2011-06-02 01:56:04 +00:00
|
|
|
import Types.Remote
|
2011-06-30 17:16:57 +00:00
|
|
|
import qualified Git
|
2011-04-28 00:06:07 +00:00
|
|
|
import Config
|
2013-03-13 20:16:01 +00:00
|
|
|
import Config.Cost
|
2011-10-04 04:40:47 +00:00
|
|
|
import Annex.Content
|
2013-09-07 22:38:00 +00:00
|
|
|
import Annex.UUID
|
2013-04-13 22:10:49 +00:00
|
|
|
import Annex.Ssh
|
2011-08-17 00:49:54 +00:00
|
|
|
import Remote.Helper.Special
|
2015-08-17 14:42:14 +00:00
|
|
|
import Remote.Helper.Messages
|
2019-02-20 19:55:01 +00:00
|
|
|
import Remote.Helper.ExportImport
|
2018-02-28 16:09:03 +00:00
|
|
|
import Types.Export
|
2020-01-10 18:10:20 +00:00
|
|
|
import Types.ProposedAccepted
|
2014-03-18 16:55:08 +00:00
|
|
|
import Remote.Rsync.RsyncUrl
|
2011-04-28 00:06:07 +00:00
|
|
|
import Crypto
|
2012-09-19 18:28:32 +00:00
|
|
|
import Utility.Rsync
|
2013-02-15 17:33:36 +00:00
|
|
|
import Utility.CopyFile
|
2015-04-03 20:48:30 +00:00
|
|
|
import Messages.Progress
|
2013-03-28 21:03:04 +00:00
|
|
|
import Utility.Metered
|
2016-08-03 16:37:12 +00:00
|
|
|
import Types.Transfer
|
2014-02-11 18:06:50 +00:00
|
|
|
import Types.Creds
|
2015-01-28 20:51:40 +00:00
|
|
|
import Annex.DirHashes
|
2017-12-31 20:08:31 +00:00
|
|
|
import Utility.Tmp.Dir
|
2017-08-18 02:11:31 +00:00
|
|
|
import Utility.SshHost
|
2020-01-14 16:35:08 +00:00
|
|
|
import Annex.SpecialRemote.Config
|
2011-04-28 00:06:07 +00:00
|
|
|
|
2014-02-11 19:29:56 +00:00
|
|
|
import qualified Data.Map as M
|
|
|
|
|
2011-12-31 08:11:39 +00:00
|
|
|
remote :: RemoteType
|
2020-01-14 16:35:08 +00:00
|
|
|
remote = specialRemoteType $ RemoteType
|
2017-09-07 17:45:31 +00:00
|
|
|
{ typename = "rsync"
|
|
|
|
, enumerate = const (findSpecialRemotes "rsyncurl")
|
|
|
|
, generate = gen
|
2020-01-20 19:13:49 +00:00
|
|
|
, configParser = mkRemoteConfigParser $ rsyncRemoteConfigs ++
|
|
|
|
[ optionalStringParser rsyncUrlField
|
2020-01-20 19:20:04 +00:00
|
|
|
(FieldDesc "(required) url or hostname:/directory for rsync to use")
|
2020-01-14 16:35:08 +00:00
|
|
|
]
|
2017-09-07 17:45:31 +00:00
|
|
|
, setup = rsyncSetup
|
2018-02-28 16:09:03 +00:00
|
|
|
, exportSupported = exportIsSupported
|
2019-02-20 19:55:01 +00:00
|
|
|
, importSupported = importUnsupported
|
2017-09-07 17:45:31 +00:00
|
|
|
}
|
2011-04-28 00:06:07 +00:00
|
|
|
|
2020-01-14 16:35:08 +00:00
|
|
|
shellEscapeField :: RemoteConfigField
|
|
|
|
shellEscapeField = Accepted "shellescape"
|
|
|
|
|
|
|
|
rsyncUrlField :: RemoteConfigField
|
|
|
|
rsyncUrlField = Accepted "rsyncurl"
|
|
|
|
|
|
|
|
gen :: Git.Repo -> UUID -> ParsedRemoteConfig -> RemoteGitConfig -> RemoteStateHandle -> Annex (Maybe Remote)
|
add RemoteStateHandle
This solves the problem of sameas remotes trampling over per-remote
state. Used for:
* per-remote state, of course
* per-remote metadata, also of course
* per-remote content identifiers, because two remote implementations
could in theory generate the same content identifier for two different
peices of content
While chunk logs are per-remote data, they don't use this, because the
number and size of chunks stored is a common property across sameas
remotes.
External special remote had a complication, where it was theoretically
possible for a remote to send SETSTATE or GETSTATE during INITREMOTE or
EXPORTSUPPORTED. Since the uuid of the remote is typically generate in
Remote.setup, it would only be possible to pass a Maybe
RemoteStateHandle into it, and it would otherwise have to construct its
own. Rather than go that route, I decided to send an ERROR in this case.
It seems unlikely that any existing external special remote will be
affected. They would have to make up a git-annex key, and set state for
some reason during INITREMOTE. I can imagine such a hack, but it doesn't
seem worth complicating the code in such an ugly way to support it.
Unfortunately, both TestRemote and Annex.Import needed the Remote
to have a new field added that holds its RemoteStateHandle.
2019-10-14 16:33:27 +00:00
|
|
|
gen r u c gc rs = do
|
2013-01-01 17:52:47 +00:00
|
|
|
cst <- remoteCost gc expensiveRemoteCost
|
2013-09-08 18:54:28 +00:00
|
|
|
(transport, url) <- rsyncTransport gc $
|
2016-11-16 01:29:54 +00:00
|
|
|
fromMaybe (giveup "missing rsyncurl") $ remoteAnnexRsyncUrl gc
|
2013-09-08 18:54:28 +00:00
|
|
|
let o = genRsyncOpts c gc transport url
|
|
|
|
let islocal = rsyncUrlIsPath $ rsyncUrl o
|
2014-08-03 20:54:57 +00:00
|
|
|
return $ Just $ specialRemote' specialcfg c
|
2014-08-03 21:31:10 +00:00
|
|
|
(simplyPrepare $ fileStorer $ store o)
|
|
|
|
(simplyPrepare $ fileRetriever $ retrieve o)
|
run Preparer to get Remover and CheckPresent actions
This will allow special remotes to eg, open a http connection and reuse it,
while checking if chunks are present, or removing chunks.
S3 and WebDAV both need this to support chunks with reasonable speed.
Note that a special remote might want to cache a http connection across
multiple requests. A simple case of this is that CheckPresent is typically
called before Store or Remove. A remote using this interface can certianly
use a Preparer that eg, uses a MVar to cache a http connection.
However, it's up to the remote to then deal with things like stale or
stalled http connections when eg, doing a series of downloads from a remote
and other places. There could be long delays between calls to a remote,
which could lead to eg, http connection stalls; the machine might even
move to a new network, etc.
It might be nice to improve this interface later to allow
the simple case without needing to handle the full complex case.
One way to do it would be to have a `Transaction SpecialRemote cache`,
where SpecialRemote contains methods for Storer, Retriever, Remover, and
CheckPresent, that all expect to be passed a `cache`.
2014-08-06 18:28:36 +00:00
|
|
|
(simplyPrepare $ remove o)
|
|
|
|
(simplyPrepare $ checkKey r o)
|
2012-07-22 17:48:50 +00:00
|
|
|
Remote
|
|
|
|
{ uuid = u
|
|
|
|
, cost = cst
|
|
|
|
, name = Git.repoDescribe r
|
2014-08-03 20:54:57 +00:00
|
|
|
, storeKey = storeKeyDummy
|
|
|
|
, retrieveKeyFile = retreiveKeyFileDummy
|
2012-07-22 17:48:50 +00:00
|
|
|
, retrieveKeyFileCheap = retrieveCheap o
|
2018-06-21 15:35:27 +00:00
|
|
|
, retrievalSecurityPolicy = RetrievalAllKeysSecure
|
run Preparer to get Remover and CheckPresent actions
This will allow special remotes to eg, open a http connection and reuse it,
while checking if chunks are present, or removing chunks.
S3 and WebDAV both need this to support chunks with reasonable speed.
Note that a special remote might want to cache a http connection across
multiple requests. A simple case of this is that CheckPresent is typically
called before Store or Remove. A remote using this interface can certianly
use a Preparer that eg, uses a MVar to cache a http connection.
However, it's up to the remote to then deal with things like stale or
stalled http connections when eg, doing a series of downloads from a remote
and other places. There could be long delays between calls to a remote,
which could lead to eg, http connection stalls; the machine might even
move to a new network, etc.
It might be nice to improve this interface later to allow
the simple case without needing to handle the full complex case.
One way to do it would be to have a `Transaction SpecialRemote cache`,
where SpecialRemote contains methods for Storer, Retriever, Remover, and
CheckPresent, that all expect to be passed a `cache`.
2014-08-06 18:28:36 +00:00
|
|
|
, removeKey = removeKeyDummy
|
2015-10-08 19:01:38 +00:00
|
|
|
, lockContent = Nothing
|
run Preparer to get Remover and CheckPresent actions
This will allow special remotes to eg, open a http connection and reuse it,
while checking if chunks are present, or removing chunks.
S3 and WebDAV both need this to support chunks with reasonable speed.
Note that a special remote might want to cache a http connection across
multiple requests. A simple case of this is that CheckPresent is typically
called before Store or Remove. A remote using this interface can certianly
use a Preparer that eg, uses a MVar to cache a http connection.
However, it's up to the remote to then deal with things like stale or
stalled http connections when eg, doing a series of downloads from a remote
and other places. There could be long delays between calls to a remote,
which could lead to eg, http connection stalls; the machine might even
move to a new network, etc.
It might be nice to improve this interface later to allow
the simple case without needing to handle the full complex case.
One way to do it would be to have a `Transaction SpecialRemote cache`,
where SpecialRemote contains methods for Storer, Retriever, Remover, and
CheckPresent, that all expect to be passed a `cache`.
2014-08-06 18:28:36 +00:00
|
|
|
, checkPresent = checkPresentDummy
|
2014-08-06 17:45:19 +00:00
|
|
|
, checkPresentCheap = False
|
2019-01-30 18:55:28 +00:00
|
|
|
, exportActions = ExportActions
|
2018-02-28 16:09:03 +00:00
|
|
|
{ storeExport = storeExportM o
|
|
|
|
, retrieveExport = retrieveExportM o
|
|
|
|
, removeExport = removeExportM o
|
|
|
|
, checkPresentExport = checkPresentExportM o
|
|
|
|
, removeExportDirectory = Just (removeExportDirectoryM o)
|
|
|
|
, renameExport = renameExportM o
|
|
|
|
}
|
2019-02-20 19:55:01 +00:00
|
|
|
, importActions = importUnsupported
|
2012-07-22 17:48:50 +00:00
|
|
|
, whereisKey = Nothing
|
2013-10-11 20:03:18 +00:00
|
|
|
, remoteFsck = Nothing
|
2013-10-27 19:38:59 +00:00
|
|
|
, repairRepo = Nothing
|
2013-11-02 20:37:28 +00:00
|
|
|
, config = c
|
2018-06-04 18:31:55 +00:00
|
|
|
, getRepo = return r
|
2013-01-01 17:52:47 +00:00
|
|
|
, gitconfig = gc
|
2013-03-15 23:16:13 +00:00
|
|
|
, localpath = if islocal
|
2012-07-22 17:48:50 +00:00
|
|
|
then Just $ rsyncUrl o
|
|
|
|
else Nothing
|
2012-08-26 19:39:02 +00:00
|
|
|
, readonly = False
|
2018-08-30 15:12:18 +00:00
|
|
|
, appendonly = False
|
2014-01-13 18:41:10 +00:00
|
|
|
, availability = if islocal then LocallyAvailable else GloballyAvailable
|
2012-07-22 17:48:50 +00:00
|
|
|
, remotetype = remote
|
2014-08-10 18:52:58 +00:00
|
|
|
, mkUnavailable = return Nothing
|
2014-10-21 18:36:09 +00:00
|
|
|
, getInfo = return [("url", url)]
|
2014-12-08 17:40:15 +00:00
|
|
|
, claimUrl = Nothing
|
2014-12-11 19:32:42 +00:00
|
|
|
, checkUrl = Nothing
|
add RemoteStateHandle
This solves the problem of sameas remotes trampling over per-remote
state. Used for:
* per-remote state, of course
* per-remote metadata, also of course
* per-remote content identifiers, because two remote implementations
could in theory generate the same content identifier for two different
peices of content
While chunk logs are per-remote data, they don't use this, because the
number and size of chunks stored is a common property across sameas
remotes.
External special remote had a complication, where it was theoretically
possible for a remote to send SETSTATE or GETSTATE during INITREMOTE or
EXPORTSUPPORTED. Since the uuid of the remote is typically generate in
Remote.setup, it would only be possible to pass a Maybe
RemoteStateHandle into it, and it would otherwise have to construct its
own. Rather than go that route, I decided to send an ERROR in this case.
It seems unlikely that any existing external special remote will be
affected. They would have to make up a git-annex key, and set state for
some reason during INITREMOTE. I can imagine such a hack, but it doesn't
seem worth complicating the code in such an ugly way to support it.
Unfortunately, both TestRemote and Annex.Import needed the Remote
to have a new field added that holds its RemoteStateHandle.
2019-10-14 16:33:27 +00:00
|
|
|
, remoteStateHandle = rs
|
2012-07-22 17:48:50 +00:00
|
|
|
}
|
2014-08-03 20:54:57 +00:00
|
|
|
where
|
|
|
|
specialcfg = (specialRemoteCfg c)
|
|
|
|
-- Rsync displays its own progress.
|
|
|
|
{ displayProgress = False }
|
2013-09-08 18:54:28 +00:00
|
|
|
|
2020-01-20 19:13:49 +00:00
|
|
|
-- Things used by genRsyncOpts
|
|
|
|
rsyncRemoteConfigs :: [RemoteConfigFieldParser]
|
|
|
|
rsyncRemoteConfigs =
|
|
|
|
[ yesNoParser shellEscapeField True
|
2020-01-20 19:20:04 +00:00
|
|
|
(FieldDesc "avoid usual shell escaping (not recommended)")
|
2020-01-20 19:13:49 +00:00
|
|
|
]
|
|
|
|
|
2020-01-14 16:35:08 +00:00
|
|
|
genRsyncOpts :: ParsedRemoteConfig -> RemoteGitConfig -> Annex [CommandParam] -> RsyncUrl -> RsyncOpts
|
2014-02-02 20:06:34 +00:00
|
|
|
genRsyncOpts c gc transport url = RsyncOpts
|
|
|
|
{ rsyncUrl = url
|
2019-06-13 15:09:55 +00:00
|
|
|
, rsyncOptions = appendtransport $ opts []
|
|
|
|
, rsyncUploadOptions = appendtransport $
|
|
|
|
opts (remoteAnnexRsyncUploadOptions gc)
|
|
|
|
, rsyncDownloadOptions = appendtransport $
|
|
|
|
opts (remoteAnnexRsyncDownloadOptions gc)
|
2020-01-14 16:35:08 +00:00
|
|
|
, rsyncShellEscape = fromMaybe True (getRemoteConfigValue shellEscapeField c)
|
2014-02-02 20:06:34 +00:00
|
|
|
}
|
2012-11-11 04:51:07 +00:00
|
|
|
where
|
2019-06-13 15:09:55 +00:00
|
|
|
appendtransport l = (++ l) <$> transport
|
2014-02-02 20:06:34 +00:00
|
|
|
opts specificopts = map Param $ filter safe $
|
|
|
|
remoteAnnexRsyncOptions gc ++ specificopts
|
2013-01-01 17:52:47 +00:00
|
|
|
safe opt
|
2012-11-11 04:51:07 +00:00
|
|
|
-- Don't allow user to pass --delete to rsync;
|
|
|
|
-- that could cause it to delete other keys
|
|
|
|
-- in the same hash bucket as a key it sends.
|
2013-01-01 17:52:47 +00:00
|
|
|
| opt == "--delete" = False
|
|
|
|
| opt == "--delete-excluded" = False
|
2012-11-11 04:51:07 +00:00
|
|
|
| otherwise = True
|
2013-09-08 18:54:28 +00:00
|
|
|
|
2019-06-13 15:09:55 +00:00
|
|
|
rsyncTransport :: RemoteGitConfig -> RsyncUrl -> Annex (Annex [CommandParam], RsyncUrl)
|
2014-02-21 17:06:39 +00:00
|
|
|
rsyncTransport gc url
|
|
|
|
| rsyncUrlIsShell url =
|
2019-06-13 15:09:55 +00:00
|
|
|
(\transport -> return (rsyncShell <$> transport, url)) =<<
|
2013-09-08 18:54:28 +00:00
|
|
|
case fromNull ["ssh"] (remoteAnnexRsyncTransport gc) of
|
|
|
|
"ssh":sshopts -> do
|
|
|
|
let (port, sshopts') = sshReadPort sshopts
|
2017-08-18 02:11:31 +00:00
|
|
|
userhost = either error id $ mkSshHost $
|
|
|
|
takeWhile (/= ':') url
|
2019-06-13 15:09:55 +00:00
|
|
|
return $ (Param "ssh":) <$> sshOptions ConsumeStdin
|
2015-02-12 19:44:10 +00:00
|
|
|
(userhost, port) gc
|
2013-09-08 18:54:28 +00:00
|
|
|
(map Param $ loginopt ++ sshopts')
|
2019-06-13 15:09:55 +00:00
|
|
|
"rsh":rshopts -> return $ pure $ map Param $ "rsh" :
|
2013-09-08 18:54:28 +00:00
|
|
|
loginopt ++ rshopts
|
2016-11-16 01:29:54 +00:00
|
|
|
rsh -> giveup $ "Unknown Rsync transport: "
|
2013-09-08 18:54:28 +00:00
|
|
|
++ unwords rsh
|
2019-06-13 15:09:55 +00:00
|
|
|
| otherwise = return (pure [], url)
|
2013-09-08 18:54:28 +00:00
|
|
|
where
|
2014-02-21 17:06:39 +00:00
|
|
|
login = case separate (=='@') url of
|
|
|
|
(_h, "") -> Nothing
|
|
|
|
(l, _) -> Just l
|
2013-04-13 22:10:49 +00:00
|
|
|
loginopt = maybe [] (\l -> ["-l",l]) login
|
2013-09-08 18:54:28 +00:00
|
|
|
fromNull as xs = if null xs then as else xs
|
2011-04-28 00:30:43 +00:00
|
|
|
|
2017-02-07 18:35:58 +00:00
|
|
|
rsyncSetup :: SetupStage -> Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID)
|
|
|
|
rsyncSetup _ mu _ c gc = do
|
2013-09-07 22:38:00 +00:00
|
|
|
u <- maybe (liftIO genUUID) return mu
|
2011-04-28 00:06:07 +00:00
|
|
|
-- verify configuration is sane
|
2020-01-10 18:10:20 +00:00
|
|
|
let url = maybe (giveup "Specify rsyncurl=") fromProposedAccepted $
|
2020-01-14 16:35:08 +00:00
|
|
|
M.lookup rsyncUrlField c
|
2016-05-23 21:27:15 +00:00
|
|
|
(c', _encsetup) <- encryptionSetup c gc
|
2011-04-28 00:06:07 +00:00
|
|
|
|
|
|
|
-- The rsyncurl is stored in git config, not only in this remote's
|
|
|
|
-- persistant state, so it can vary between hosts.
|
2018-03-27 16:41:57 +00:00
|
|
|
gitConfigSpecialRemote u c' [("rsyncurl", url)]
|
2013-09-07 22:38:00 +00:00
|
|
|
return (c', u)
|
2011-04-28 00:06:07 +00:00
|
|
|
|
2014-08-03 21:31:10 +00:00
|
|
|
{- To send a single key is slightly tricky; need to build up a temporary
|
|
|
|
- directory structure to pass to rsync so it can create the hash
|
|
|
|
- directories.
|
|
|
|
-
|
|
|
|
- This would not be necessary if the hash directory structure used locally
|
|
|
|
- was always the same as that used on the rsync remote. So if that's ever
|
|
|
|
- unified, this gets nicer.
|
|
|
|
- (When we have the right hash directory structure, we can just
|
|
|
|
- pass --include=X --include=X/Y --include=X/Y/file --exclude=*)
|
|
|
|
-}
|
|
|
|
store :: RsyncOpts -> Key -> FilePath -> MeterUpdate -> Annex Bool
|
2018-02-28 16:09:03 +00:00
|
|
|
store o k src meterupdate = storeGeneric o meterupdate basedest populatedest
|
|
|
|
where
|
2019-12-11 18:12:22 +00:00
|
|
|
basedest = fromRawFilePath $ Prelude.head (keyPaths k)
|
2018-02-28 16:09:03 +00:00
|
|
|
populatedest dest = liftIO $ if canrename
|
2014-08-03 21:31:10 +00:00
|
|
|
then do
|
|
|
|
rename src dest
|
|
|
|
return True
|
|
|
|
else createLinkOrCopy src dest
|
2018-02-28 16:09:03 +00:00
|
|
|
{- If the key being sent is encrypted or chunked, the file
|
|
|
|
- containing its content is a temp file, and so can be
|
|
|
|
- renamed into place. Otherwise, the file is the annexed
|
|
|
|
- object file, and has to be copied or hard linked into place. -}
|
|
|
|
canrename = isEncKey k || isChunkKey k
|
|
|
|
|
|
|
|
storeGeneric :: RsyncOpts -> MeterUpdate -> FilePath -> (FilePath -> Annex Bool) -> Annex Bool
|
|
|
|
storeGeneric o meterupdate basedest populatedest = withRsyncScratchDir $ \tmp -> do
|
|
|
|
let dest = tmp </> basedest
|
|
|
|
liftIO $ createDirectoryIfMissing True $ parentDir dest
|
|
|
|
ok <- populatedest dest
|
2014-08-03 21:31:10 +00:00
|
|
|
ps <- sendParams
|
|
|
|
if ok
|
|
|
|
then showResumable $ rsyncRemote Upload o (Just meterupdate) $ ps ++
|
2015-06-01 17:52:23 +00:00
|
|
|
Param "--recursive" : partialParams ++
|
2014-08-03 21:31:10 +00:00
|
|
|
-- tmp/ to send contents of tmp dir
|
2015-06-01 17:52:23 +00:00
|
|
|
[ File $ addTrailingPathSeparator tmp
|
2014-08-03 21:31:10 +00:00
|
|
|
, Param $ rsyncUrl o
|
|
|
|
]
|
|
|
|
else return False
|
2011-04-28 00:06:07 +00:00
|
|
|
|
2014-08-03 21:31:10 +00:00
|
|
|
retrieve :: RsyncOpts -> FilePath -> Key -> MeterUpdate -> Annex ()
|
|
|
|
retrieve o f k p =
|
2018-02-28 16:09:03 +00:00
|
|
|
unlessM (rsyncRetrieveKey o k f (Just p)) $
|
2016-11-16 01:29:54 +00:00
|
|
|
giveup "rsync failed"
|
2012-01-20 17:23:11 +00:00
|
|
|
|
2015-04-14 20:35:10 +00:00
|
|
|
retrieveCheap :: RsyncOpts -> Key -> AssociatedFile -> FilePath -> Annex Bool
|
2018-02-28 16:09:03 +00:00
|
|
|
retrieveCheap o k _af f = ifM (preseedTmp k f) ( rsyncRetrieveKey o k f Nothing , return False )
|
2011-04-28 00:06:07 +00:00
|
|
|
|
run Preparer to get Remover and CheckPresent actions
This will allow special remotes to eg, open a http connection and reuse it,
while checking if chunks are present, or removing chunks.
S3 and WebDAV both need this to support chunks with reasonable speed.
Note that a special remote might want to cache a http connection across
multiple requests. A simple case of this is that CheckPresent is typically
called before Store or Remove. A remote using this interface can certianly
use a Preparer that eg, uses a MVar to cache a http connection.
However, it's up to the remote to then deal with things like stale or
stalled http connections when eg, doing a series of downloads from a remote
and other places. There could be long delays between calls to a remote,
which could lead to eg, http connection stalls; the machine might even
move to a new network, etc.
It might be nice to improve this interface later to allow
the simple case without needing to handle the full complex case.
One way to do it would be to have a `Transaction SpecialRemote cache`,
where SpecialRemote contains methods for Storer, Retriever, Remover, and
CheckPresent, that all expect to be passed a `cache`.
2014-08-06 18:28:36 +00:00
|
|
|
remove :: RsyncOpts -> Remover
|
2018-02-28 16:09:03 +00:00
|
|
|
remove o k = removeGeneric o includes
|
|
|
|
where
|
|
|
|
includes = concatMap use dirHashes
|
2019-12-11 18:12:22 +00:00
|
|
|
use h = let dir = fromRawFilePath (h def k) in
|
2018-02-28 16:09:03 +00:00
|
|
|
[ parentDir dir
|
|
|
|
, dir
|
|
|
|
-- match content directory and anything in it
|
2019-12-18 20:45:03 +00:00
|
|
|
, dir </> fromRawFilePath (keyFile k) </> "***"
|
2018-02-28 16:09:03 +00:00
|
|
|
]
|
|
|
|
|
|
|
|
{- An empty directory is rsynced to make it delete. Everything is excluded,
|
|
|
|
- except for the specified includes. Due to the way rsync traverses
|
|
|
|
- directories, the includes must match both the file to be deleted, and
|
|
|
|
- its parent directories, but not their other contents. -}
|
|
|
|
removeGeneric :: RsyncOpts -> [String] -> Annex Bool
|
|
|
|
removeGeneric o includes = do
|
2013-05-09 17:49:47 +00:00
|
|
|
ps <- sendParams
|
2019-06-13 15:09:55 +00:00
|
|
|
opts <- rsyncOptions o
|
2013-05-09 17:49:47 +00:00
|
|
|
withRsyncScratchDir $ \tmp -> liftIO $ do
|
|
|
|
{- Send an empty directory to rysnc to make it delete. -}
|
2019-06-13 15:09:55 +00:00
|
|
|
rsync $ opts ++ ps ++
|
2013-05-09 17:49:47 +00:00
|
|
|
map (\s -> Param $ "--include=" ++ s) includes ++
|
|
|
|
[ Param "--exclude=*" -- exclude everything else
|
2015-06-01 17:52:23 +00:00
|
|
|
, Param "--quiet", Param "--delete", Param "--recursive"
|
|
|
|
] ++ partialParams ++
|
2018-02-28 16:09:03 +00:00
|
|
|
[ Param $ addTrailingPathSeparator tmp
|
2013-05-09 17:49:47 +00:00
|
|
|
, Param $ rsyncUrl o
|
|
|
|
]
|
2011-04-28 00:06:07 +00:00
|
|
|
|
run Preparer to get Remover and CheckPresent actions
This will allow special remotes to eg, open a http connection and reuse it,
while checking if chunks are present, or removing chunks.
S3 and WebDAV both need this to support chunks with reasonable speed.
Note that a special remote might want to cache a http connection across
multiple requests. A simple case of this is that CheckPresent is typically
called before Store or Remove. A remote using this interface can certianly
use a Preparer that eg, uses a MVar to cache a http connection.
However, it's up to the remote to then deal with things like stale or
stalled http connections when eg, doing a series of downloads from a remote
and other places. There could be long delays between calls to a remote,
which could lead to eg, http connection stalls; the machine might even
move to a new network, etc.
It might be nice to improve this interface later to allow
the simple case without needing to handle the full complex case.
One way to do it would be to have a `Transaction SpecialRemote cache`,
where SpecialRemote contains methods for Storer, Retriever, Remover, and
CheckPresent, that all expect to be passed a `cache`.
2014-08-06 18:28:36 +00:00
|
|
|
checkKey :: Git.Repo -> RsyncOpts -> CheckPresent
|
2014-08-06 17:45:19 +00:00
|
|
|
checkKey r o k = do
|
2015-08-17 14:42:14 +00:00
|
|
|
showChecking r
|
2018-02-28 16:09:03 +00:00
|
|
|
checkPresentGeneric o (rsyncUrls o k)
|
|
|
|
|
|
|
|
checkPresentGeneric :: RsyncOpts -> [RsyncUrl] -> Annex Bool
|
2019-06-13 15:09:55 +00:00
|
|
|
checkPresentGeneric o rsyncurls = do
|
|
|
|
opts <- rsyncOptions o
|
2011-12-02 19:50:27 +00:00
|
|
|
-- note: Does not currently differentiate between rsync failing
|
2011-04-28 00:06:07 +00:00
|
|
|
-- to connect, and the file not being present.
|
2018-02-28 16:09:03 +00:00
|
|
|
untilTrue rsyncurls $ \u ->
|
2012-11-11 04:51:07 +00:00
|
|
|
liftIO $ catchBoolIO $ do
|
|
|
|
withQuietOutput createProcessSuccess $
|
2019-06-13 15:09:55 +00:00
|
|
|
proc "rsync" $ toCommand $ opts ++ [Param u]
|
2012-11-11 04:51:07 +00:00
|
|
|
return True
|
2011-04-28 00:06:07 +00:00
|
|
|
|
2018-02-28 16:09:03 +00:00
|
|
|
storeExportM :: RsyncOpts -> FilePath -> Key -> ExportLocation -> MeterUpdate -> Annex Bool
|
|
|
|
storeExportM o src _k loc meterupdate =
|
|
|
|
storeGeneric o meterupdate basedest populatedest
|
|
|
|
where
|
2019-11-26 19:27:22 +00:00
|
|
|
basedest = fromRawFilePath (fromExportLocation loc)
|
2018-02-28 16:09:03 +00:00
|
|
|
populatedest = liftIO . createLinkOrCopy src
|
|
|
|
|
|
|
|
retrieveExportM :: RsyncOpts -> Key -> ExportLocation -> FilePath -> MeterUpdate -> Annex Bool
|
|
|
|
retrieveExportM o _k loc dest p = rsyncRetrieve o [rsyncurl] dest (Just p)
|
|
|
|
where
|
2019-11-26 19:27:22 +00:00
|
|
|
rsyncurl = mkRsyncUrl o (fromRawFilePath (fromExportLocation loc))
|
2018-02-28 16:09:03 +00:00
|
|
|
|
|
|
|
checkPresentExportM :: RsyncOpts -> Key -> ExportLocation -> Annex Bool
|
|
|
|
checkPresentExportM o _k loc = checkPresentGeneric o [rsyncurl]
|
|
|
|
where
|
2019-11-26 19:27:22 +00:00
|
|
|
rsyncurl = mkRsyncUrl o (fromRawFilePath (fromExportLocation loc))
|
2018-02-28 16:09:03 +00:00
|
|
|
|
|
|
|
removeExportM :: RsyncOpts -> Key -> ExportLocation -> Annex Bool
|
|
|
|
removeExportM o _k loc =
|
2019-11-26 19:27:22 +00:00
|
|
|
removeGeneric o $ includes $ fromRawFilePath $ fromExportLocation loc
|
2018-02-28 16:09:03 +00:00
|
|
|
where
|
|
|
|
includes f = f : case upFrom f of
|
|
|
|
Nothing -> []
|
|
|
|
Just f' -> includes f'
|
|
|
|
|
|
|
|
removeExportDirectoryM :: RsyncOpts -> ExportDirectory -> Annex Bool
|
|
|
|
removeExportDirectoryM o ed = removeGeneric o (allbelow d : includes d)
|
|
|
|
where
|
2019-11-26 19:27:22 +00:00
|
|
|
d = fromRawFilePath $ fromExportDirectory ed
|
2018-02-28 16:09:03 +00:00
|
|
|
allbelow f = f </> "***"
|
|
|
|
includes f = f : case upFrom f of
|
|
|
|
Nothing -> []
|
|
|
|
Just f' -> includes f'
|
|
|
|
|
2019-03-11 16:44:12 +00:00
|
|
|
renameExportM :: RsyncOpts -> Key -> ExportLocation -> ExportLocation -> Annex (Maybe Bool)
|
|
|
|
renameExportM _ _ _ _ = return Nothing
|
2018-02-28 16:09:03 +00:00
|
|
|
|
2011-04-28 00:06:07 +00:00
|
|
|
{- Rsync params to enable resumes of sending files safely,
|
|
|
|
- ensure that files are only moved into place once complete
|
|
|
|
-}
|
2015-06-01 17:52:23 +00:00
|
|
|
partialParams :: [CommandParam]
|
|
|
|
partialParams = [Param "--partial", Param "--partial-dir=.rsync-partial"]
|
2011-04-28 00:06:07 +00:00
|
|
|
|
2013-05-09 17:49:47 +00:00
|
|
|
{- When sending files from crippled filesystems, the permissions can be all
|
|
|
|
- messed up, and it's better to use the default permissions on the
|
|
|
|
- destination. -}
|
|
|
|
sendParams :: Annex [CommandParam]
|
|
|
|
sendParams = ifM crippledFileSystem
|
|
|
|
( return [rsyncUseDestinationPermissions]
|
|
|
|
, return []
|
|
|
|
)
|
|
|
|
|
2011-04-28 00:06:07 +00:00
|
|
|
{- Runs an action in an empty scratch directory that can be used to build
|
|
|
|
- up trees for rsync. -}
|
2013-09-24 21:25:47 +00:00
|
|
|
withRsyncScratchDir :: (FilePath -> Annex a) -> Annex a
|
2011-04-28 00:06:07 +00:00
|
|
|
withRsyncScratchDir a = do
|
2014-02-26 20:52:56 +00:00
|
|
|
t <- fromRepo gitAnnexTmpObjectDir
|
2015-07-30 17:29:45 +00:00
|
|
|
withTmpDirIn t "rsynctmp" a
|
2011-04-28 00:06:07 +00:00
|
|
|
|
2018-02-28 16:09:03 +00:00
|
|
|
rsyncRetrieve :: RsyncOpts -> [RsyncUrl] -> FilePath -> Maybe MeterUpdate -> Annex Bool
|
|
|
|
rsyncRetrieve o rsyncurls dest meterupdate =
|
|
|
|
showResumable $ untilTrue rsyncurls $ \u -> rsyncRemote Download o meterupdate
|
2013-04-11 21:15:45 +00:00
|
|
|
-- use inplace when retrieving to support resuming
|
|
|
|
[ Param "--inplace"
|
|
|
|
, Param u
|
2013-05-14 17:24:15 +00:00
|
|
|
, File dest
|
2013-04-11 21:15:45 +00:00
|
|
|
]
|
|
|
|
|
2018-02-28 16:09:03 +00:00
|
|
|
rsyncRetrieveKey :: RsyncOpts -> Key -> FilePath -> Maybe MeterUpdate -> Annex Bool
|
|
|
|
rsyncRetrieveKey o k dest meterupdate = rsyncRetrieve o (rsyncUrls o k) dest meterupdate
|
|
|
|
|
2013-12-02 16:53:39 +00:00
|
|
|
showResumable :: Annex Bool -> Annex Bool
|
|
|
|
showResumable a = ifM a
|
|
|
|
( return True
|
|
|
|
, do
|
|
|
|
showLongNote "rsync failed -- run git annex again to resume file transfer"
|
|
|
|
return False
|
|
|
|
)
|
|
|
|
|
2014-02-02 20:06:34 +00:00
|
|
|
rsyncRemote :: Direction -> RsyncOpts -> Maybe MeterUpdate -> [CommandParam] -> Annex Bool
|
2015-04-03 20:48:30 +00:00
|
|
|
rsyncRemote direction o m params = do
|
2011-07-19 18:07:23 +00:00
|
|
|
showOutput -- make way for progress bar
|
2019-06-13 15:09:55 +00:00
|
|
|
opts <- mkopts
|
|
|
|
let ps = opts ++ Param "--progress" : params
|
2015-04-03 20:48:30 +00:00
|
|
|
case m of
|
|
|
|
Nothing -> liftIO $ rsync ps
|
|
|
|
Just meter -> do
|
2015-04-04 18:34:03 +00:00
|
|
|
oh <- mkOutputHandler
|
|
|
|
liftIO $ rsyncProgress oh meter ps
|
2012-11-11 04:51:07 +00:00
|
|
|
where
|
2019-06-13 15:09:55 +00:00
|
|
|
mkopts
|
2014-02-02 20:06:34 +00:00
|
|
|
| direction == Download = rsyncDownloadOptions o
|
|
|
|
| otherwise = rsyncUploadOptions o
|