d9b4bf4224
I anticipate lots of external special remote programs will neglect implementing this. Still, it's the right thing to do to assume that some of them may write files out of order. Probably most external special remotes will not be used with a proxy. When someone is using one with a proxy, they can always get it fixed to send ORDERED.
415 lines
14 KiB
Haskell
415 lines
14 KiB
Haskell
{- A remote that is only accessible by rsync.
|
|
-
|
|
- Copyright 2011-2020 Joey Hess <id@joeyh.name>
|
|
-
|
|
- Licensed under the GNU AGPL version 3 or higher.
|
|
-}
|
|
|
|
{-# LANGUAGE CPP, OverloadedStrings #-}
|
|
|
|
module Remote.Rsync (
|
|
remote,
|
|
store,
|
|
retrieve,
|
|
remove,
|
|
checkKey,
|
|
withRsyncScratchDir,
|
|
rsyncRemoteConfigs,
|
|
genRsyncOpts,
|
|
RsyncOpts,
|
|
probeRsyncProtectsArgs,
|
|
) where
|
|
|
|
import Annex.Common
|
|
import Types.Remote
|
|
import qualified Git
|
|
import Config
|
|
import Config.Cost
|
|
import Annex.Content
|
|
import Annex.UUID
|
|
import Annex.Ssh
|
|
import Annex.Perms
|
|
import Remote.Helper.Special
|
|
import Remote.Helper.ExportImport
|
|
import Remote.Helper.Path
|
|
import Types.Export
|
|
import Types.ProposedAccepted
|
|
import Remote.Rsync.RsyncUrl
|
|
import Crypto
|
|
import Utility.Rsync
|
|
import Utility.CopyFile
|
|
import Utility.Process.Transcript
|
|
import Messages.Progress
|
|
import Utility.Metered
|
|
import Types.Transfer
|
|
import Types.Creds
|
|
import Annex.DirHashes
|
|
import Utility.Tmp.Dir
|
|
import Utility.SshHost
|
|
import Annex.SpecialRemote.Config
|
|
import Annex.Verify
|
|
import qualified Utility.RawFilePath as R
|
|
|
|
import qualified Data.Map as M
|
|
import qualified Data.List.NonEmpty as NE
|
|
|
|
remote :: RemoteType
|
|
remote = specialRemoteType $ RemoteType
|
|
{ typename = "rsync"
|
|
, enumerate = const (findSpecialRemotes "rsyncurl")
|
|
, generate = gen
|
|
, configParser = mkRemoteConfigParser $ rsyncRemoteConfigs ++
|
|
[ optionalStringParser rsyncUrlField
|
|
(FieldDesc "(required) url or hostname:/directory for rsync to use")
|
|
]
|
|
, setup = rsyncSetup
|
|
, exportSupported = exportIsSupported
|
|
, importSupported = importUnsupported
|
|
, thirdPartyPopulated = False
|
|
}
|
|
|
|
shellEscapeField :: RemoteConfigField
|
|
shellEscapeField = Accepted "shellescape"
|
|
|
|
rsyncUrlField :: RemoteConfigField
|
|
rsyncUrlField = Accepted "rsyncurl"
|
|
|
|
gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> RemoteStateHandle -> Annex (Maybe Remote)
|
|
gen r u rc gc rs = do
|
|
c <- parsedRemoteConfig remote rc
|
|
cst <- remoteCost gc c expensiveRemoteCost
|
|
(transport, url) <- rsyncTransport gc $
|
|
fromMaybe (giveup "missing rsyncurl") $ remoteAnnexRsyncUrl gc
|
|
protectsargs <- liftIO probeRsyncProtectsArgs
|
|
let o = genRsyncOpts protectsargs c gc transport url
|
|
let islocal = rsyncUrlIsPath $ rsyncUrl o
|
|
return $ Just $ specialRemote c
|
|
(fileStorer $ store o)
|
|
(fileRetriever $ retrieve o)
|
|
(remove o)
|
|
(checkKey o)
|
|
Remote
|
|
{ uuid = u
|
|
, cost = cst
|
|
, name = Git.repoDescribe r
|
|
, storeKey = storeKeyDummy
|
|
, retrieveKeyFile = retrieveKeyFileDummy
|
|
, retrieveKeyFileInOrder = pure True
|
|
, retrieveKeyFileCheap = Just (retrieveCheap o)
|
|
, retrievalSecurityPolicy = RetrievalAllKeysSecure
|
|
, removeKey = removeKeyDummy
|
|
, lockContent = Nothing
|
|
, checkPresent = checkPresentDummy
|
|
, checkPresentCheap = False
|
|
, exportActions = ExportActions
|
|
{ storeExport = storeExportM o
|
|
, retrieveExport = retrieveExportM o
|
|
, removeExport = removeExportM o
|
|
, checkPresentExport = checkPresentExportM o
|
|
, removeExportDirectory = Just (removeExportDirectoryM o)
|
|
, renameExport = Just $ renameExportM o
|
|
}
|
|
, importActions = importUnsupported
|
|
, whereisKey = Nothing
|
|
, remoteFsck = Nothing
|
|
, repairRepo = Nothing
|
|
, config = c
|
|
, getRepo = return r
|
|
, gitconfig = gc
|
|
, localpath = if islocal
|
|
then Just $ rsyncUrl o
|
|
else Nothing
|
|
, readonly = False
|
|
, appendonly = False
|
|
, untrustworthy = False
|
|
, availability = checkPathAvailability islocal (rsyncUrl o)
|
|
, remotetype = remote
|
|
, mkUnavailable = return Nothing
|
|
, getInfo = return [("url", url)]
|
|
, claimUrl = Nothing
|
|
, checkUrl = Nothing
|
|
, remoteStateHandle = rs
|
|
}
|
|
|
|
-- | Since 3.2.4, rsync protects filenames from being exposed to the shell.
|
|
newtype RsyncProtectsArgs = RsyncProtectsArgs Bool
|
|
|
|
probeRsyncProtectsArgs :: IO RsyncProtectsArgs
|
|
probeRsyncProtectsArgs = do
|
|
(helpoutput, _) <- processTranscript "rsync" ["--help"] Nothing
|
|
-- The --old-args option was added to disable the new arg
|
|
-- protection, so use it to detect when that feature is supported
|
|
-- by rsync, rather than parsing versions.
|
|
return (RsyncProtectsArgs $ "--old-args" `isInfixOf` helpoutput)
|
|
|
|
|
|
-- Things used by genRsyncOpts
|
|
rsyncRemoteConfigs :: [RemoteConfigFieldParser]
|
|
rsyncRemoteConfigs =
|
|
[ yesNoParser shellEscapeField (Just True)
|
|
(FieldDesc "set to no to avoid usual shell escaping (not recommended)")
|
|
]
|
|
|
|
genRsyncOpts :: RsyncProtectsArgs -> ParsedRemoteConfig -> RemoteGitConfig -> Annex [CommandParam] -> RsyncUrl -> RsyncOpts
|
|
genRsyncOpts (RsyncProtectsArgs protectsargs) c gc transport url = RsyncOpts
|
|
{ rsyncUrl = url
|
|
, rsyncOptions = appendtransport $ opts []
|
|
, rsyncUploadOptions = appendtransport $
|
|
opts (remoteAnnexRsyncUploadOptions gc)
|
|
, rsyncDownloadOptions = appendtransport $
|
|
opts (remoteAnnexRsyncDownloadOptions gc)
|
|
, rsyncShellEscape = if protectsargs
|
|
then False
|
|
else fromMaybe True (getRemoteConfigValue shellEscapeField c)
|
|
}
|
|
where
|
|
appendtransport l = (++ l) <$> transport
|
|
opts specificopts = map Param $ filter safe $
|
|
remoteAnnexRsyncOptions gc ++ specificopts
|
|
safe opt
|
|
-- Don't allow user to pass --delete to rsync;
|
|
-- that could cause it to delete other keys
|
|
-- in the same hash bucket as a key it sends.
|
|
| opt == "--delete" = False
|
|
| opt == "--delete-excluded" = False
|
|
| otherwise = True
|
|
|
|
rsyncTransport :: RemoteGitConfig -> RsyncUrl -> Annex (Annex [CommandParam], RsyncUrl)
|
|
rsyncTransport gc url
|
|
| rsyncUrlIsShell url =
|
|
(\transport -> return (rsyncShell <$> transport, url)) =<<
|
|
case fromNull ["ssh"] (remoteAnnexRsyncTransport gc) of
|
|
"ssh":sshopts -> do
|
|
let (port, sshopts') = sshReadPort sshopts
|
|
userhost = either giveup id $ mkSshHost $
|
|
takeWhile (/= ':') url
|
|
return $ (Param "ssh":) <$> sshOptions ConsumeStdin
|
|
(userhost, port) gc
|
|
(map Param $ loginopt ++ sshopts')
|
|
"rsh":rshopts -> return $ pure $ map Param $ "rsh" :
|
|
loginopt ++ rshopts
|
|
rsh -> giveup $ "Unknown Rsync transport: "
|
|
++ unwords rsh
|
|
| otherwise = return (pure [], url)
|
|
where
|
|
login = case separate (=='@') url of
|
|
(_h, "") -> Nothing
|
|
(l, _) -> Just l
|
|
loginopt = maybe [] (\l -> ["-l",l]) login
|
|
fromNull as xs = if null xs then as else xs
|
|
|
|
rsyncSetup :: SetupStage -> Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID)
|
|
rsyncSetup _ mu _ c gc = do
|
|
u <- maybe (liftIO genUUID) return mu
|
|
-- verify configuration is sane
|
|
let url = maybe (giveup "Specify rsyncurl=") fromProposedAccepted $
|
|
M.lookup rsyncUrlField c
|
|
(c', _encsetup) <- encryptionSetup c gc
|
|
|
|
-- The rsyncurl is stored in git config, not only in this remote's
|
|
-- persistent state, so it can vary between hosts.
|
|
gitConfigSpecialRemote u c' [("rsyncurl", url)]
|
|
return (c', u)
|
|
|
|
{- To send a single key is slightly tricky; need to build up a temporary
|
|
- directory structure to pass to rsync so it can create the hash
|
|
- directories.
|
|
-
|
|
- This would not be necessary if the hash directory structure used locally
|
|
- was always the same as that used on the rsync remote. So if that's ever
|
|
- unified, this gets nicer.
|
|
- (When we have the right hash directory structure, we can just
|
|
- pass --include=X --include=X/Y --include=X/Y/file --exclude=*)
|
|
-}
|
|
store :: RsyncOpts -> Key -> FilePath -> MeterUpdate -> Annex ()
|
|
store o k src meterupdate = storeGeneric o meterupdate basedest populatedest
|
|
where
|
|
basedest = fromRawFilePath $ NE.head (keyPaths k)
|
|
populatedest dest = liftIO $ if canrename
|
|
then do
|
|
R.rename (toRawFilePath src) (toRawFilePath dest)
|
|
return True
|
|
else createLinkOrCopy (toRawFilePath src) (toRawFilePath dest)
|
|
{- If the key being sent is encrypted or chunked, the file
|
|
- containing its content is a temp file, and so can be
|
|
- renamed into place. Otherwise, the file is the annexed
|
|
- object file, and has to be copied or hard linked into place. -}
|
|
canrename = isEncKey k || isChunkKey k
|
|
|
|
storeGeneric :: RsyncOpts -> MeterUpdate -> FilePath -> (FilePath -> Annex Bool) -> Annex ()
|
|
storeGeneric o meterupdate basedest populatedest =
|
|
unlessM (storeGeneric' o meterupdate basedest populatedest) $
|
|
giveup "failed to rsync content"
|
|
|
|
storeGeneric' :: RsyncOpts -> MeterUpdate -> FilePath -> (FilePath -> Annex Bool) -> Annex Bool
|
|
storeGeneric' o meterupdate basedest populatedest = withRsyncScratchDir $ \tmp -> do
|
|
let dest = tmp </> basedest
|
|
createAnnexDirectory (parentDir (toRawFilePath dest))
|
|
ok <- populatedest dest
|
|
ps <- sendParams
|
|
if ok
|
|
then showResumable $ rsyncRemote Upload o (Just meterupdate) $ ps ++
|
|
Param "--recursive" : partialParams ++
|
|
-- tmp/ to send contents of tmp dir
|
|
[ File $ addTrailingPathSeparator tmp
|
|
, Param $ rsyncUrl o
|
|
]
|
|
else return False
|
|
|
|
retrieve :: RsyncOpts -> RawFilePath -> Key -> MeterUpdate -> Annex ()
|
|
retrieve o f k p = rsyncRetrieveKey o k (fromRawFilePath f) (Just p)
|
|
|
|
retrieveCheap :: RsyncOpts -> Key -> AssociatedFile -> FilePath -> Annex ()
|
|
retrieveCheap o k _af f = ifM (preseedTmp k f)
|
|
( rsyncRetrieveKey o k f Nothing
|
|
, giveup "cannot preseed rsync with existing content"
|
|
)
|
|
|
|
remove :: RsyncOpts -> Remover
|
|
remove o _proof k = removeGeneric o includes
|
|
where
|
|
includes = concatMap use dirHashes
|
|
use h = let dir = fromRawFilePath (h def k) in
|
|
[ fromRawFilePath (parentDir (toRawFilePath dir))
|
|
, dir
|
|
-- match content directory and anything in it
|
|
, dir </> fromRawFilePath (keyFile k) </> "***"
|
|
]
|
|
|
|
{- An empty directory is rsynced to make it delete. Everything is excluded,
|
|
- except for the specified includes. Due to the way rsync traverses
|
|
- directories, the includes must match both the file to be deleted, and
|
|
- its parent directories, but not their other contents. -}
|
|
removeGeneric :: RsyncOpts -> [String] -> Annex ()
|
|
removeGeneric o includes = do
|
|
ps <- sendParams
|
|
opts <- rsyncOptions o
|
|
ok <- withRsyncScratchDir $ \tmp -> liftIO $ do
|
|
{- Send an empty directory to rsync to make it delete. -}
|
|
rsync $ opts ++ ps ++
|
|
map (\s -> Param $ "--include=" ++ s) includes ++
|
|
[ Param "--exclude=*" -- exclude everything else
|
|
, Param "--quiet", Param "--delete", Param "--recursive"
|
|
] ++ partialParams ++
|
|
[ Param $ addTrailingPathSeparator tmp
|
|
, Param $ rsyncUrl o
|
|
]
|
|
unless ok $
|
|
giveup "rsync failed"
|
|
|
|
checkKey :: RsyncOpts -> CheckPresent
|
|
checkKey o k = checkPresentGeneric o (rsyncUrls o k)
|
|
|
|
checkPresentGeneric :: RsyncOpts -> [RsyncUrl] -> Annex Bool
|
|
checkPresentGeneric o rsyncurls = do
|
|
opts <- rsyncOptions o
|
|
-- note: Does not currently differentiate between rsync failing
|
|
-- to connect, and the file not being present.
|
|
untilTrue rsyncurls $ \u ->
|
|
liftIO $ catchBoolIO $ withNullHandle $ \nullh ->
|
|
let p = (proc "rsync" $ toCommand $ opts ++ [Param u])
|
|
{ std_out = UseHandle nullh
|
|
, std_err = UseHandle nullh
|
|
}
|
|
in withCreateProcess p $ \_ _ _ -> checkSuccessProcess
|
|
|
|
storeExportM :: RsyncOpts -> FilePath -> Key -> ExportLocation -> MeterUpdate -> Annex ()
|
|
storeExportM o src _k loc meterupdate =
|
|
storeGeneric o meterupdate basedest populatedest
|
|
where
|
|
basedest = fromRawFilePath (fromExportLocation loc)
|
|
populatedest = liftIO . createLinkOrCopy (toRawFilePath src) . toRawFilePath
|
|
|
|
retrieveExportM :: RsyncOpts -> Key -> ExportLocation -> FilePath -> MeterUpdate -> Annex Verification
|
|
retrieveExportM o k loc dest p =
|
|
verifyKeyContentIncrementally AlwaysVerify k $ \iv ->
|
|
tailVerify iv (toRawFilePath dest) $
|
|
rsyncRetrieve o [rsyncurl] dest (Just p)
|
|
where
|
|
rsyncurl = mkRsyncUrl o (fromRawFilePath (fromExportLocation loc))
|
|
|
|
checkPresentExportM :: RsyncOpts -> Key -> ExportLocation -> Annex Bool
|
|
checkPresentExportM o _k loc = checkPresentGeneric o [rsyncurl]
|
|
where
|
|
rsyncurl = mkRsyncUrl o (fromRawFilePath (fromExportLocation loc))
|
|
|
|
removeExportM :: RsyncOpts -> Key -> ExportLocation -> Annex ()
|
|
removeExportM o _k loc =
|
|
removeGeneric o $ map fromRawFilePath $
|
|
includes $ fromExportLocation loc
|
|
where
|
|
includes f = f : case upFrom f of
|
|
Nothing -> []
|
|
Just f' -> includes f'
|
|
|
|
removeExportDirectoryM :: RsyncOpts -> ExportDirectory -> Annex ()
|
|
removeExportDirectoryM o ed = removeGeneric o (allbelow d : includes d)
|
|
where
|
|
d = fromRawFilePath $ fromExportDirectory ed
|
|
allbelow f = f </> "***"
|
|
includes f = f : case upFrom (toRawFilePath f) of
|
|
Nothing -> []
|
|
Just f' -> includes (fromRawFilePath f')
|
|
|
|
renameExportM :: RsyncOpts -> Key -> ExportLocation -> ExportLocation -> Annex (Maybe ())
|
|
renameExportM _ _ _ _ = return Nothing
|
|
|
|
{- Rsync params to enable resumes of sending files safely,
|
|
- ensure that files are only moved into place once complete
|
|
-}
|
|
partialParams :: [CommandParam]
|
|
partialParams = [Param "--partial", Param "--partial-dir=.rsync-partial"]
|
|
|
|
{- When sending files from crippled filesystems, the permissions can be all
|
|
- messed up, and it's better to use the default permissions on the
|
|
- destination. -}
|
|
sendParams :: Annex [CommandParam]
|
|
sendParams = ifM crippledFileSystem
|
|
( return [rsyncUseDestinationPermissions]
|
|
, return []
|
|
)
|
|
|
|
{- Runs an action in an empty scratch directory that can be used to build
|
|
- up trees for rsync. -}
|
|
withRsyncScratchDir :: (FilePath -> Annex a) -> Annex a
|
|
withRsyncScratchDir a = do
|
|
t <- fromRawFilePath <$> fromRepo gitAnnexTmpObjectDir
|
|
withTmpDirIn t "rsynctmp" a
|
|
|
|
rsyncRetrieve :: RsyncOpts -> [RsyncUrl] -> FilePath -> Maybe MeterUpdate -> Annex ()
|
|
rsyncRetrieve o rsyncurls dest meterupdate =
|
|
unlessM go $
|
|
giveup "rsync failed"
|
|
where
|
|
go = showResumable $ untilTrue rsyncurls $ \u -> rsyncRemote Download o meterupdate
|
|
-- use inplace when retrieving to support resuming
|
|
[ Param "--inplace"
|
|
, Param u
|
|
, File dest
|
|
]
|
|
|
|
rsyncRetrieveKey :: RsyncOpts -> Key -> FilePath -> Maybe MeterUpdate -> Annex ()
|
|
rsyncRetrieveKey o k dest meterupdate =
|
|
rsyncRetrieve o (rsyncUrls o k) dest meterupdate
|
|
|
|
showResumable :: Annex Bool -> Annex Bool
|
|
showResumable a = ifM a
|
|
( return True
|
|
, do
|
|
showLongNote "rsync failed -- run git annex again to resume file transfer"
|
|
return False
|
|
)
|
|
|
|
rsyncRemote :: Direction -> RsyncOpts -> Maybe MeterUpdate -> [CommandParam] -> Annex Bool
|
|
rsyncRemote direction o m params = do
|
|
opts <- mkopts
|
|
let ps = opts ++ Param "--progress" : params
|
|
case m of
|
|
Nothing -> liftIO $ rsync ps
|
|
Just meter -> do
|
|
oh <- mkOutputHandlerQuiet
|
|
liftIO $ rsyncProgress oh meter ps
|
|
where
|
|
mkopts
|
|
| direction == Download = rsyncDownloadOptions o
|
|
| otherwise = rsyncUploadOptions o
|