git-annex/Remote/Helper/Ssh.hs

341 lines
12 KiB
Haskell
Raw Normal View History

{- git-annex remote access with ssh and git-annex-shell
-
- Copyright 2011-2024 Joey Hess <id@joeyh.name>
-
2018-03-09 17:48:10 +00:00
- Licensed under the GNU AGPL version 3 or higher.
-}
{-# LANGUAGE OverloadedStrings #-}
2012-01-10 19:29:10 +00:00
module Remote.Helper.Ssh where
import Annex.Common
2014-05-16 20:08:20 +00:00
import qualified Annex
import qualified Git
2011-12-14 19:30:14 +00:00
import qualified Git.Url
import Annex.UUID
import Annex.Ssh
2014-01-26 20:32:55 +00:00
import CmdLine.GitAnnexShell.Fields (Field, fieldName)
import qualified CmdLine.GitAnnexShell.Fields as Fields
import Remote.Helper.Messages
import Utility.Metered
import Utility.Rsync
import Utility.SshHost
import Utility.Debug
import Types.Remote
import Types.Transfer
import Config
import qualified P2P.Protocol as P2P
import qualified P2P.IO as P2P
2018-03-09 17:48:10 +00:00
import qualified P2P.Annex as P2P
import Control.Concurrent.STM
toRepo :: ConsumeStdin -> Git.Repo -> RemoteGitConfig -> SshCommand -> Annex (FilePath, [CommandParam])
toRepo cs r gc remotecmd = do
let host = maybe
(giveup "bad ssh url")
(either giveup id . mkSshHost)
(Git.Url.hostuser r)
sshCommand cs (host, Git.Url.port r) gc remotecmd
2011-04-09 18:26:32 +00:00
{- Generates parameters to run a git-annex-shell command on a remote
- repository. -}
git_annex_shell :: ConsumeStdin -> Git.Repo -> String -> [CommandParam] -> [(Field, String)] -> Annex (Maybe (FilePath, [CommandParam]))
git_annex_shell cs r command params fields
| not $ Git.repoIsUrl r = do
dir <- liftIO $ absPath (Git.repoPath r)
shellopts <- getshellopts dir
return $ Just (shellcmd, shellopts ++ fieldopts)
2011-04-09 18:26:32 +00:00
| Git.repoIsSsh r = do
2014-05-16 20:08:20 +00:00
gc <- Annex.getRemoteGitConfig r
u <- getRepoUUID r
shellopts <- getshellopts (Git.repoPath r)
let sshcmd = unwords $
fromMaybe shellcmd (remoteAnnexShell gc)
: map shellEscape (toCommand shellopts) ++
uuidcheck u ++
map shellEscape (toCommand fieldopts)
Just <$> toRepo cs r gc sshcmd
2011-04-09 18:26:32 +00:00
| otherwise = return Nothing
2012-11-11 04:51:07 +00:00
where
shellcmd = "git-annex-shell"
getshellopts dir = do
debugenabled <- Annex.getRead Annex.debugenabled
debugselector <- Annex.getRead Annex.debugselector
let params' = case (debugenabled, debugselector) of
(True, NoDebugSelector) -> Param "--debug" : params
_ -> params
return (Param command : File (fromRawFilePath dir) : params')
2012-11-11 04:51:07 +00:00
uuidcheck NoUUID = []
uuidcheck u@(UUID _) = ["--uuid", fromUUID u]
2012-11-11 04:51:07 +00:00
fieldopts
| null fields = []
| otherwise = fieldsep : map fieldopt fields ++ [fieldsep]
fieldsep = Param "--"
fieldopt (field, value) = Param $
fieldName field ++ "=" ++ value
2011-04-09 18:26:32 +00:00
{- Uses a supplied function (such as boolSystem) to run a git-annex-shell
- command on a remote.
-
- Or, if the remote does not support running remote commands, returns
- a specified error value. -}
onRemote
:: ConsumeStdin
-> Git.Repo
-> (FilePath -> [CommandParam] -> Annex a, Annex a)
2011-04-09 18:26:32 +00:00
-> String
-> [CommandParam]
-> [(Field, String)]
2011-04-09 18:26:32 +00:00
-> Annex a
onRemote cs r (with, errorval) command params fields = do
s <- git_annex_shell cs r command params fields
2011-04-09 18:26:32 +00:00
case s of
Just (c, ps) -> with c ps
Nothing -> errorval
{- Checks if a remote contains a key. -}
inAnnex :: Git.Repo -> Key -> Annex Bool
inAnnex r k = onRemote NoConsumeStdin r (runcheck, cantCheck r) "inannex"
[Param $ serializeKey k] []
where
runcheck c p = liftIO $ dispatch =<< safeSystem c p
dispatch ExitSuccess = return True
dispatch (ExitFailure 1) = return False
dispatch _ = cantCheck r
toward SafeDropProof expiry checking Added Maybe POSIXTime to SafeDropProof, which gets set when the proof is based on a LockedCopy. If there are several LockedCopies, it uses the closest expiry time. That is not optimal, it may be that the proof expires based on one LockedCopy but another one has not expired. But that seems unlikely to really happen, and anyway the user can just re-run a drop if it fails due to expiry. Pass the SafeDropProof to removeKey, which is responsible for checking it for expiry in situations where that could be a problem. Which really only means in Remote.Git. Made Remote.Git check expiry when dropping from a local remote. Checking expiry when dropping from a P2P remote is not yet implemented. P2P.Protocol.remove has SafeDropProof plumbed through to it for that purpose. Fixing the remaining 2 build warnings should complete this work. Note that the use of a POSIXTime here means that if the clock gets set forward while git-annex is in the middle of a drop, it may say that dropping took too long. That seems ok. Less ok is that if the clock gets turned back a sufficient amount (eg 5 minutes), proof expiry won't be noticed. It might be better to use the Monotonic clock, but that doesn't advance when a laptop is suspended, and while there is the linux Boottime clock, that is not available on other systems. Perhaps a combination of POSIXTime and the Monotonic clock could detect laptop suspension and also detect clock being turned back? There is a potential future flag day where p2pDefaultLockContentRetentionDuration is not assumed, but is probed using the P2P protocol, and peers that don't support it can no longer produce a LockedCopy. Until that happens, when git-annex is communicating with older peers there is a risk of data loss when a ssh connection closes during LOCKCONTENT.
2024-07-04 16:23:46 +00:00
{- Removes a key from a remote using the legacy git-annex-shell dropkey,
- rather than the P2P protocol.
-
- The proof is not checked for expire on the remote, so this should only
- be used by remotes that do not have lockContent return a LockedCopy. -}
dropKey :: Git.Repo -> Maybe SafeDropProof -> Key -> Annex ()
dropKey r proof key = unlessM (dropKey' r proof key) $
2020-05-14 18:08:09 +00:00
giveup "unable to remove key from remote"
toward SafeDropProof expiry checking Added Maybe POSIXTime to SafeDropProof, which gets set when the proof is based on a LockedCopy. If there are several LockedCopies, it uses the closest expiry time. That is not optimal, it may be that the proof expires based on one LockedCopy but another one has not expired. But that seems unlikely to really happen, and anyway the user can just re-run a drop if it fails due to expiry. Pass the SafeDropProof to removeKey, which is responsible for checking it for expiry in situations where that could be a problem. Which really only means in Remote.Git. Made Remote.Git check expiry when dropping from a local remote. Checking expiry when dropping from a P2P remote is not yet implemented. P2P.Protocol.remove has SafeDropProof plumbed through to it for that purpose. Fixing the remaining 2 build warnings should complete this work. Note that the use of a POSIXTime here means that if the clock gets set forward while git-annex is in the middle of a drop, it may say that dropping took too long. That seems ok. Less ok is that if the clock gets turned back a sufficient amount (eg 5 minutes), proof expiry won't be noticed. It might be better to use the Monotonic clock, but that doesn't advance when a laptop is suspended, and while there is the linux Boottime clock, that is not available on other systems. Perhaps a combination of POSIXTime and the Monotonic clock could detect laptop suspension and also detect clock being turned back? There is a potential future flag day where p2pDefaultLockContentRetentionDuration is not assumed, but is probed using the P2P protocol, and peers that don't support it can no longer produce a LockedCopy. Until that happens, when git-annex is communicating with older peers there is a risk of data loss when a ssh connection closes during LOCKCONTENT.
2024-07-04 16:23:46 +00:00
dropKey' :: Git.Repo -> Maybe SafeDropProof -> Key -> Annex Bool
dropKey' r _proof key = onRemote NoConsumeStdin r (\f p -> liftIO (boolSystem f p), return False) "dropkey"
[ Param "--quiet", Param "--force"
, Param $ serializeKey key
]
[]
rsyncHelper :: OutputHandler -> Maybe MeterUpdate -> [CommandParam] -> Annex Bool
rsyncHelper oh m params = do
unless (quietMode oh) $
showOutput -- make way for progress bar
a <- case m of
Nothing -> return $ rsync params
Just meter -> return $ rsyncProgress oh meter params
ifM (liftIO a)
( return True
, do
showLongNote "rsync failed -- run git annex again to resume file transfer"
return False
)
{- Generates rsync parameters that ssh to the remote and asks it
- to either receive or send the key's content. -}
rsyncParamsRemote :: Remote -> Direction -> Key -> FilePath -> Annex [CommandParam]
rsyncParamsRemote r direction key file = do
u <- getUUID
repo <- getRepo r
Just (shellcmd, shellparams) <- git_annex_shell ConsumeStdin repo
(if direction == Download then "sendkey" else "recvkey")
[ Param $ serializeKey key ]
[(Fields.remoteUUID, fromUUID u)]
-- Convert the ssh command into rsync command line.
let eparam = rsyncShell (Param shellcmd:shellparams)
o <- rsyncParams r direction
2013-09-26 03:19:01 +00:00
return $ if direction == Download
then o ++ rsyncopts eparam dummy (File file)
else o ++ rsyncopts eparam (File file) dummy
where
rsyncopts ps source dest
| end ps == [dashdash] = ps ++ [source, dest]
| otherwise = ps ++ [dashdash, source, dest]
dashdash = Param "--"
{- The rsync shell parameter controls where rsync
- goes, so the source/dest parameter can be a dummy value,
- that just enables remote rsync mode.
2023-03-14 02:39:16 +00:00
- For maximum compatibility with some patched rsyncs,
- the dummy value needs to still contain a hostname,
- even though this hostname will never be used. -}
dummy = Param "dummy:"
-- --inplace to resume partial files
--
-- Only use --perms when not on a crippled file system, as rsync
-- will fail trying to restore file perms onto a filesystem that does not
-- support them.
rsyncParams :: Remote -> Direction -> Annex [CommandParam]
rsyncParams r direction = do
crippled <- crippledFileSystem
return $ map Param $ catMaybes
[ Just "--progress"
, Just "--inplace"
, if crippled then Nothing else Just "--perms"
]
++ remoteAnnexRsyncOptions gc ++ dps
where
dps
| direction == Download = remoteAnnexRsyncDownloadOptions gc
| otherwise = remoteAnnexRsyncUploadOptions gc
gc = gitconfig r
2015-10-09 20:55:41 +00:00
-- A connection over ssh or locally to git-annex shell,
-- speaking the P2P protocol.
type P2PShellConnection = P2P.ClosableConnection
(P2P.RunState, P2P.P2PConnection, ProcessHandle)
closeP2PShellConnection :: P2PShellConnection -> IO (P2PShellConnection, Maybe ExitCode)
closeP2PShellConnection P2P.ClosedConnection = return (P2P.ClosedConnection, Nothing)
closeP2PShellConnection (P2P.OpenConnection (_st, conn, pid)) =
-- mask async exceptions, avoid cleanup being interrupted
uninterruptibleMask_ $ do
P2P.closeConnection conn
exitcode <- waitForProcess pid
return (P2P.ClosedConnection, Just exitcode)
-- Pool of connections to git-annex-shell p2pstdio.
type P2PShellConnectionPool = TVar (Maybe P2PShellConnectionPoolState)
data P2PShellConnectionPoolState
= P2PShellConnections [P2PShellConnection]
-- Remotes using an old version of git-annex-shell don't support P2P
| P2PShellUnsupported
mkP2PShellConnectionPool :: Annex P2PShellConnectionPool
mkP2PShellConnectionPool = liftIO $ newTVarIO Nothing
-- Takes a connection from the pool, if any are available, otherwise
-- tries to open a new one.
getP2PShellConnection :: Remote -> P2PShellConnectionPool -> Annex (Maybe P2PShellConnection)
getP2PShellConnection r connpool = getexistingconn >>= \case
Nothing -> return Nothing
Just Nothing -> openP2PShellConnection r connpool
Just (Just c) -> return (Just c)
where
getexistingconn = liftIO $ atomically $ readTVar connpool >>= \case
Just P2PShellUnsupported -> return Nothing
Just (P2PShellConnections (c:cs)) -> do
writeTVar connpool (Just (P2PShellConnections cs))
return (Just (Just c))
Just (P2PShellConnections []) -> return (Just Nothing)
Nothing -> return (Just Nothing)
-- Add a connection to the pool, unless it's closed.
storeP2PShellConnection :: P2PShellConnectionPool -> P2PShellConnection -> IO ()
storeP2PShellConnection _ P2P.ClosedConnection = return ()
storeP2PShellConnection connpool conn = atomically $ modifyTVar' connpool $ \case
Just (P2PShellConnections cs) -> Just (P2PShellConnections (conn:cs))
_ -> Just (P2PShellConnections [conn])
-- Try to open a P2PShellConnection.
-- The new connection is not added to the pool, so it's available
-- for the caller to use.
-- If the remote does not support the P2P protocol, that's remembered in
-- the connection pool.
openP2PShellConnection :: Remote -> P2PShellConnectionPool -> Annex (Maybe P2PShellConnection)
openP2PShellConnection r connpool =
openP2PShellConnection' r P2P.maxProtocolVersion mempty >>= \case
Just conn -> return (Just conn)
Nothing -> do
liftIO $ rememberunsupported
return Nothing
where
rememberunsupported = atomically $
modifyTVar' connpool $
maybe (Just P2PShellUnsupported) Just
openP2PShellConnection' :: Remote -> P2P.ProtocolVersion -> P2P.Bypass -> Annex (Maybe P2PShellConnection)
openP2PShellConnection' r maxprotoversion bypass = do
u <- getUUID
let ps = [Param (fromUUID u)]
repo <- getRepo r
git_annex_shell ConsumeStdin repo "p2pstdio" ps [] >>= \case
Nothing -> return Nothing
Just (cmd, params) -> start cmd params
where
start cmd params = liftIO $ do
(Just from, Just to, Nothing, pid) <- createProcess $
(proc cmd (toCommand params))
{ std_in = CreatePipe
, std_out = CreatePipe
}
pidnum <- getPid pid
let conn = P2P.P2PConnection
{ P2P.connRepo = Nothing
, P2P.connCheckAuth = const False
, P2P.connIhdl = P2P.P2PHandle to
, P2P.connOhdl = P2P.P2PHandle from
, P2P.connIdent = P2P.ConnIdent $
Just $ "git-annex-shell connection " ++ show pidnum
}
runst <- P2P.mkRunState P2P.Client
let c = P2P.OpenConnection (runst, conn, pid)
-- When the connection is successful, the remote
-- will send an AUTH_SUCCESS with its uuid.
let proto = P2P.postAuth $ do
P2P.negotiateProtocolVersion maxprotoversion
P2P.sendBypass bypass
tryNonAsync (P2P.runNetProto runst conn proto) >>= \case
Right (Right (Just theiruuid)) | theiruuid == uuid r ->
return $ Just c
_ -> do
(cclosed, exitcode) <- closeP2PShellConnection c
-- ssh exits 255 when unable to connect to
2022-01-03 16:38:29 +00:00
-- server.
if exitcode == Just (ExitFailure 255)
then return (Just cclosed)
else return Nothing
2018-03-09 17:48:10 +00:00
-- Runs a P2P Proto action on a remote when it supports that,
-- otherwise the fallback action.
runProto :: Remote -> P2PShellConnectionPool -> Annex a -> P2P.Proto a -> Annex (Maybe a)
runProto r connpool onerr proto = Just <$>
(getP2PShellConnection r connpool >>= maybe onerr go)
2018-03-09 17:48:10 +00:00
where
go c = do
(c', v) <- runProtoConn proto c
case v of
Just res -> do
liftIO $ storeP2PShellConnection connpool c'
2018-03-09 17:48:10 +00:00
return res
Nothing -> onerr
2018-03-09 17:48:10 +00:00
runProtoConn :: P2P.Proto a -> P2PShellConnection -> Annex (P2PShellConnection, Maybe a)
2018-03-09 17:48:10 +00:00
runProtoConn _ P2P.ClosedConnection = return (P2P.ClosedConnection, Nothing)
runProtoConn a conn@(P2P.OpenConnection (runst, c, _)) = do
P2P.runFullProto runst c a >>= \case
2018-03-09 17:48:10 +00:00
Right r -> return (conn, Just r)
-- When runFullProto fails, the connection is no longer
-- usable, so close it.
Left e -> do
warning $ UnquotedString $ "Lost connection (" ++ P2P.describeProtoFailure e ++ ")"
conn' <- fst <$> liftIO (closeP2PShellConnection conn)
2018-03-09 17:48:10 +00:00
return (conn', Nothing)
-- Allocates a P2P shell connection from the pool, and runs the action with
-- it, returning the connection to the pool once the action is done.
2018-03-09 17:48:10 +00:00
--
-- If the remote does not support the P2P protocol, runs the fallback
-- action instead.
withP2PShellConnection
2018-03-09 17:48:10 +00:00
:: Remote
-> P2PShellConnectionPool
2018-03-09 17:48:10 +00:00
-> Annex a
-> (P2PShellConnection -> Annex (P2PShellConnection, a))
2018-03-09 17:48:10 +00:00
-> Annex a
withP2PShellConnection r connpool fallback a = bracketOnError get cache go
2018-03-09 17:48:10 +00:00
where
get = getP2PShellConnection r connpool
cache (Just conn) = liftIO $ storeP2PShellConnection connpool conn
2018-03-09 17:48:10 +00:00
cache Nothing = return ()
go (Just conn) = do
(conn', res) <- a conn
cache (Just conn')
return res
go Nothing = fallback