git-annex/Remote/Rsync.hs
Joey Hess 10216b44d2
use NonEmpty for dirHashes
This avoids 4 uses of head.
2024-09-26 18:15:00 -04:00

414 lines
14 KiB
Haskell

{- A remote that is only accessible by rsync.
-
- Copyright 2011-2020 Joey Hess <id@joeyh.name>
-
- Licensed under the GNU AGPL version 3 or higher.
-}
{-# LANGUAGE CPP, OverloadedStrings #-}
module Remote.Rsync (
remote,
store,
retrieve,
remove,
checkKey,
withRsyncScratchDir,
rsyncRemoteConfigs,
genRsyncOpts,
RsyncOpts,
probeRsyncProtectsArgs,
) where
import Annex.Common
import Types.Remote
import qualified Git
import Config
import Config.Cost
import Annex.Content
import Annex.UUID
import Annex.Ssh
import Annex.Perms
import Remote.Helper.Special
import Remote.Helper.ExportImport
import Remote.Helper.Path
import Types.Export
import Types.ProposedAccepted
import Remote.Rsync.RsyncUrl
import Crypto
import Utility.Rsync
import Utility.CopyFile
import Utility.Process.Transcript
import Messages.Progress
import Utility.Metered
import Types.Transfer
import Types.Creds
import Annex.DirHashes
import Utility.Tmp.Dir
import Utility.SshHost
import Annex.SpecialRemote.Config
import Annex.Verify
import qualified Utility.RawFilePath as R
import qualified Data.Map as M
import qualified Data.List.NonEmpty as NE
remote :: RemoteType
remote = specialRemoteType $ RemoteType
{ typename = "rsync"
, enumerate = const (findSpecialRemotes "rsyncurl")
, generate = gen
, configParser = mkRemoteConfigParser $ rsyncRemoteConfigs ++
[ optionalStringParser rsyncUrlField
(FieldDesc "(required) url or hostname:/directory for rsync to use")
]
, setup = rsyncSetup
, exportSupported = exportIsSupported
, importSupported = importUnsupported
, thirdPartyPopulated = False
}
shellEscapeField :: RemoteConfigField
shellEscapeField = Accepted "shellescape"
rsyncUrlField :: RemoteConfigField
rsyncUrlField = Accepted "rsyncurl"
gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> RemoteStateHandle -> Annex (Maybe Remote)
gen r u rc gc rs = do
c <- parsedRemoteConfig remote rc
cst <- remoteCost gc c expensiveRemoteCost
(transport, url) <- rsyncTransport gc $
fromMaybe (giveup "missing rsyncurl") $ remoteAnnexRsyncUrl gc
protectsargs <- liftIO probeRsyncProtectsArgs
let o = genRsyncOpts protectsargs c gc transport url
let islocal = rsyncUrlIsPath $ rsyncUrl o
return $ Just $ specialRemote c
(fileStorer $ store o)
(fileRetriever $ retrieve o)
(remove o)
(checkKey o)
Remote
{ uuid = u
, cost = cst
, name = Git.repoDescribe r
, storeKey = storeKeyDummy
, retrieveKeyFile = retrieveKeyFileDummy
, retrieveKeyFileCheap = Just (retrieveCheap o)
, retrievalSecurityPolicy = RetrievalAllKeysSecure
, removeKey = removeKeyDummy
, lockContent = Nothing
, checkPresent = checkPresentDummy
, checkPresentCheap = False
, exportActions = ExportActions
{ storeExport = storeExportM o
, retrieveExport = retrieveExportM o
, removeExport = removeExportM o
, checkPresentExport = checkPresentExportM o
, removeExportDirectory = Just (removeExportDirectoryM o)
, renameExport = Just $ renameExportM o
}
, importActions = importUnsupported
, whereisKey = Nothing
, remoteFsck = Nothing
, repairRepo = Nothing
, config = c
, getRepo = return r
, gitconfig = gc
, localpath = if islocal
then Just $ rsyncUrl o
else Nothing
, readonly = False
, appendonly = False
, untrustworthy = False
, availability = checkPathAvailability islocal (rsyncUrl o)
, remotetype = remote
, mkUnavailable = return Nothing
, getInfo = return [("url", url)]
, claimUrl = Nothing
, checkUrl = Nothing
, remoteStateHandle = rs
}
-- | Since 3.2.4, rsync protects filenames from being exposed to the shell.
newtype RsyncProtectsArgs = RsyncProtectsArgs Bool
probeRsyncProtectsArgs :: IO RsyncProtectsArgs
probeRsyncProtectsArgs = do
(helpoutput, _) <- processTranscript "rsync" ["--help"] Nothing
-- The --old-args option was added to disable the new arg
-- protection, so use it to detect when that feature is supported
-- by rsync, rather than parsing versions.
return (RsyncProtectsArgs $ "--old-args" `isInfixOf` helpoutput)
-- Things used by genRsyncOpts
rsyncRemoteConfigs :: [RemoteConfigFieldParser]
rsyncRemoteConfigs =
[ yesNoParser shellEscapeField (Just True)
(FieldDesc "set to no to avoid usual shell escaping (not recommended)")
]
genRsyncOpts :: RsyncProtectsArgs -> ParsedRemoteConfig -> RemoteGitConfig -> Annex [CommandParam] -> RsyncUrl -> RsyncOpts
genRsyncOpts (RsyncProtectsArgs protectsargs) c gc transport url = RsyncOpts
{ rsyncUrl = url
, rsyncOptions = appendtransport $ opts []
, rsyncUploadOptions = appendtransport $
opts (remoteAnnexRsyncUploadOptions gc)
, rsyncDownloadOptions = appendtransport $
opts (remoteAnnexRsyncDownloadOptions gc)
, rsyncShellEscape = if protectsargs
then False
else fromMaybe True (getRemoteConfigValue shellEscapeField c)
}
where
appendtransport l = (++ l) <$> transport
opts specificopts = map Param $ filter safe $
remoteAnnexRsyncOptions gc ++ specificopts
safe opt
-- Don't allow user to pass --delete to rsync;
-- that could cause it to delete other keys
-- in the same hash bucket as a key it sends.
| opt == "--delete" = False
| opt == "--delete-excluded" = False
| otherwise = True
rsyncTransport :: RemoteGitConfig -> RsyncUrl -> Annex (Annex [CommandParam], RsyncUrl)
rsyncTransport gc url
| rsyncUrlIsShell url =
(\transport -> return (rsyncShell <$> transport, url)) =<<
case fromNull ["ssh"] (remoteAnnexRsyncTransport gc) of
"ssh":sshopts -> do
let (port, sshopts') = sshReadPort sshopts
userhost = either giveup id $ mkSshHost $
takeWhile (/= ':') url
return $ (Param "ssh":) <$> sshOptions ConsumeStdin
(userhost, port) gc
(map Param $ loginopt ++ sshopts')
"rsh":rshopts -> return $ pure $ map Param $ "rsh" :
loginopt ++ rshopts
rsh -> giveup $ "Unknown Rsync transport: "
++ unwords rsh
| otherwise = return (pure [], url)
where
login = case separate (=='@') url of
(_h, "") -> Nothing
(l, _) -> Just l
loginopt = maybe [] (\l -> ["-l",l]) login
fromNull as xs = if null xs then as else xs
rsyncSetup :: SetupStage -> Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID)
rsyncSetup _ mu _ c gc = do
u <- maybe (liftIO genUUID) return mu
-- verify configuration is sane
let url = maybe (giveup "Specify rsyncurl=") fromProposedAccepted $
M.lookup rsyncUrlField c
(c', _encsetup) <- encryptionSetup c gc
-- The rsyncurl is stored in git config, not only in this remote's
-- persistent state, so it can vary between hosts.
gitConfigSpecialRemote u c' [("rsyncurl", url)]
return (c', u)
{- To send a single key is slightly tricky; need to build up a temporary
- directory structure to pass to rsync so it can create the hash
- directories.
-
- This would not be necessary if the hash directory structure used locally
- was always the same as that used on the rsync remote. So if that's ever
- unified, this gets nicer.
- (When we have the right hash directory structure, we can just
- pass --include=X --include=X/Y --include=X/Y/file --exclude=*)
-}
store :: RsyncOpts -> Key -> FilePath -> MeterUpdate -> Annex ()
store o k src meterupdate = storeGeneric o meterupdate basedest populatedest
where
basedest = fromRawFilePath $ NE.head (keyPaths k)
populatedest dest = liftIO $ if canrename
then do
R.rename (toRawFilePath src) (toRawFilePath dest)
return True
else createLinkOrCopy (toRawFilePath src) (toRawFilePath dest)
{- If the key being sent is encrypted or chunked, the file
- containing its content is a temp file, and so can be
- renamed into place. Otherwise, the file is the annexed
- object file, and has to be copied or hard linked into place. -}
canrename = isEncKey k || isChunkKey k
storeGeneric :: RsyncOpts -> MeterUpdate -> FilePath -> (FilePath -> Annex Bool) -> Annex ()
storeGeneric o meterupdate basedest populatedest =
unlessM (storeGeneric' o meterupdate basedest populatedest) $
giveup "failed to rsync content"
storeGeneric' :: RsyncOpts -> MeterUpdate -> FilePath -> (FilePath -> Annex Bool) -> Annex Bool
storeGeneric' o meterupdate basedest populatedest = withRsyncScratchDir $ \tmp -> do
let dest = tmp </> basedest
createAnnexDirectory (parentDir (toRawFilePath dest))
ok <- populatedest dest
ps <- sendParams
if ok
then showResumable $ rsyncRemote Upload o (Just meterupdate) $ ps ++
Param "--recursive" : partialParams ++
-- tmp/ to send contents of tmp dir
[ File $ addTrailingPathSeparator tmp
, Param $ rsyncUrl o
]
else return False
retrieve :: RsyncOpts -> RawFilePath -> Key -> MeterUpdate -> Annex ()
retrieve o f k p = rsyncRetrieveKey o k (fromRawFilePath f) (Just p)
retrieveCheap :: RsyncOpts -> Key -> AssociatedFile -> FilePath -> Annex ()
retrieveCheap o k _af f = ifM (preseedTmp k f)
( rsyncRetrieveKey o k f Nothing
, giveup "cannot preseed rsync with existing content"
)
remove :: RsyncOpts -> Remover
remove o _proof k = removeGeneric o includes
where
includes = concatMap use dirHashes
use h = let dir = fromRawFilePath (h def k) in
[ fromRawFilePath (parentDir (toRawFilePath dir))
, dir
-- match content directory and anything in it
, dir </> fromRawFilePath (keyFile k) </> "***"
]
{- An empty directory is rsynced to make it delete. Everything is excluded,
- except for the specified includes. Due to the way rsync traverses
- directories, the includes must match both the file to be deleted, and
- its parent directories, but not their other contents. -}
removeGeneric :: RsyncOpts -> [String] -> Annex ()
removeGeneric o includes = do
ps <- sendParams
opts <- rsyncOptions o
ok <- withRsyncScratchDir $ \tmp -> liftIO $ do
{- Send an empty directory to rsync to make it delete. -}
rsync $ opts ++ ps ++
map (\s -> Param $ "--include=" ++ s) includes ++
[ Param "--exclude=*" -- exclude everything else
, Param "--quiet", Param "--delete", Param "--recursive"
] ++ partialParams ++
[ Param $ addTrailingPathSeparator tmp
, Param $ rsyncUrl o
]
unless ok $
giveup "rsync failed"
checkKey :: RsyncOpts -> CheckPresent
checkKey o k = checkPresentGeneric o (rsyncUrls o k)
checkPresentGeneric :: RsyncOpts -> [RsyncUrl] -> Annex Bool
checkPresentGeneric o rsyncurls = do
opts <- rsyncOptions o
-- note: Does not currently differentiate between rsync failing
-- to connect, and the file not being present.
untilTrue rsyncurls $ \u ->
liftIO $ catchBoolIO $ withNullHandle $ \nullh ->
let p = (proc "rsync" $ toCommand $ opts ++ [Param u])
{ std_out = UseHandle nullh
, std_err = UseHandle nullh
}
in withCreateProcess p $ \_ _ _ -> checkSuccessProcess
storeExportM :: RsyncOpts -> FilePath -> Key -> ExportLocation -> MeterUpdate -> Annex ()
storeExportM o src _k loc meterupdate =
storeGeneric o meterupdate basedest populatedest
where
basedest = fromRawFilePath (fromExportLocation loc)
populatedest = liftIO . createLinkOrCopy (toRawFilePath src) . toRawFilePath
retrieveExportM :: RsyncOpts -> Key -> ExportLocation -> FilePath -> MeterUpdate -> Annex Verification
retrieveExportM o k loc dest p =
verifyKeyContentIncrementally AlwaysVerify k $ \iv ->
tailVerify iv (toRawFilePath dest) $
rsyncRetrieve o [rsyncurl] dest (Just p)
where
rsyncurl = mkRsyncUrl o (fromRawFilePath (fromExportLocation loc))
checkPresentExportM :: RsyncOpts -> Key -> ExportLocation -> Annex Bool
checkPresentExportM o _k loc = checkPresentGeneric o [rsyncurl]
where
rsyncurl = mkRsyncUrl o (fromRawFilePath (fromExportLocation loc))
removeExportM :: RsyncOpts -> Key -> ExportLocation -> Annex ()
removeExportM o _k loc =
removeGeneric o $ map fromRawFilePath $
includes $ fromExportLocation loc
where
includes f = f : case upFrom f of
Nothing -> []
Just f' -> includes f'
removeExportDirectoryM :: RsyncOpts -> ExportDirectory -> Annex ()
removeExportDirectoryM o ed = removeGeneric o (allbelow d : includes d)
where
d = fromRawFilePath $ fromExportDirectory ed
allbelow f = f </> "***"
includes f = f : case upFrom (toRawFilePath f) of
Nothing -> []
Just f' -> includes (fromRawFilePath f')
renameExportM :: RsyncOpts -> Key -> ExportLocation -> ExportLocation -> Annex (Maybe ())
renameExportM _ _ _ _ = return Nothing
{- Rsync params to enable resumes of sending files safely,
- ensure that files are only moved into place once complete
-}
partialParams :: [CommandParam]
partialParams = [Param "--partial", Param "--partial-dir=.rsync-partial"]
{- When sending files from crippled filesystems, the permissions can be all
- messed up, and it's better to use the default permissions on the
- destination. -}
sendParams :: Annex [CommandParam]
sendParams = ifM crippledFileSystem
( return [rsyncUseDestinationPermissions]
, return []
)
{- Runs an action in an empty scratch directory that can be used to build
- up trees for rsync. -}
withRsyncScratchDir :: (FilePath -> Annex a) -> Annex a
withRsyncScratchDir a = do
t <- fromRawFilePath <$> fromRepo gitAnnexTmpObjectDir
withTmpDirIn t "rsynctmp" a
rsyncRetrieve :: RsyncOpts -> [RsyncUrl] -> FilePath -> Maybe MeterUpdate -> Annex ()
rsyncRetrieve o rsyncurls dest meterupdate =
unlessM go $
giveup "rsync failed"
where
go = showResumable $ untilTrue rsyncurls $ \u -> rsyncRemote Download o meterupdate
-- use inplace when retrieving to support resuming
[ Param "--inplace"
, Param u
, File dest
]
rsyncRetrieveKey :: RsyncOpts -> Key -> FilePath -> Maybe MeterUpdate -> Annex ()
rsyncRetrieveKey o k dest meterupdate =
rsyncRetrieve o (rsyncUrls o k) dest meterupdate
showResumable :: Annex Bool -> Annex Bool
showResumable a = ifM a
( return True
, do
showLongNote "rsync failed -- run git annex again to resume file transfer"
return False
)
rsyncRemote :: Direction -> RsyncOpts -> Maybe MeterUpdate -> [CommandParam] -> Annex Bool
rsyncRemote direction o m params = do
opts <- mkopts
let ps = opts ++ Param "--progress" : params
case m of
Nothing -> liftIO $ rsync ps
Just meter -> do
oh <- mkOutputHandlerQuiet
liftIO $ rsyncProgress oh meter ps
where
mkopts
| direction == Download = rsyncDownloadOptions o
| otherwise = rsyncUploadOptions o