2011-04-28 21:21:45 +00:00
|
|
|
{- A remote that provides hooks to run shell commands.
|
|
|
|
-
|
|
|
|
- Copyright 2011 Joey Hess <joey@kitenet.net>
|
|
|
|
-
|
|
|
|
- Licensed under the GNU GPL version 3 or higher.
|
|
|
|
-}
|
|
|
|
|
|
|
|
module Remote.Hook (remote) where
|
|
|
|
|
|
|
|
import qualified Data.Map as M
|
|
|
|
|
2011-10-05 20:02:51 +00:00
|
|
|
import Common.Annex
|
2011-06-02 01:56:04 +00:00
|
|
|
import Types.Remote
|
2012-08-08 20:06:01 +00:00
|
|
|
import Types.Key
|
2014-02-11 18:06:50 +00:00
|
|
|
import Types.Creds
|
2011-06-30 17:16:57 +00:00
|
|
|
import qualified Git
|
2011-04-28 21:21:45 +00:00
|
|
|
import Config
|
2013-03-13 20:16:01 +00:00
|
|
|
import Config.Cost
|
2013-09-07 22:38:00 +00:00
|
|
|
import Annex.UUID
|
2011-08-17 00:49:54 +00:00
|
|
|
import Remote.Helper.Special
|
2014-01-14 20:42:10 +00:00
|
|
|
import Utility.Env
|
2011-04-28 21:21:45 +00:00
|
|
|
|
2013-05-21 23:19:03 +00:00
|
|
|
type Action = String
|
|
|
|
type HookName = String
|
|
|
|
|
2011-12-31 08:11:39 +00:00
|
|
|
remote :: RemoteType
|
2011-04-28 21:21:45 +00:00
|
|
|
remote = RemoteType {
|
|
|
|
typename = "hook",
|
|
|
|
enumerate = findSpecialRemotes "hooktype",
|
|
|
|
generate = gen,
|
|
|
|
setup = hookSetup
|
|
|
|
}
|
|
|
|
|
2013-09-12 19:54:35 +00:00
|
|
|
gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remote)
|
2013-01-01 17:52:47 +00:00
|
|
|
gen r u c gc = do
|
|
|
|
cst <- remoteCost gc expensiveRemoteCost
|
2014-08-03 19:35:23 +00:00
|
|
|
return $ Just $ specialRemote c
|
2014-08-02 21:25:16 +00:00
|
|
|
(simplyPrepare $ store hooktype)
|
|
|
|
(simplyPrepare $ retrieve hooktype)
|
run Preparer to get Remover and CheckPresent actions
This will allow special remotes to eg, open a http connection and reuse it,
while checking if chunks are present, or removing chunks.
S3 and WebDAV both need this to support chunks with reasonable speed.
Note that a special remote might want to cache a http connection across
multiple requests. A simple case of this is that CheckPresent is typically
called before Store or Remove. A remote using this interface can certianly
use a Preparer that eg, uses a MVar to cache a http connection.
However, it's up to the remote to then deal with things like stale or
stalled http connections when eg, doing a series of downloads from a remote
and other places. There could be long delays between calls to a remote,
which could lead to eg, http connection stalls; the machine might even
move to a new network, etc.
It might be nice to improve this interface later to allow
the simple case without needing to handle the full complex case.
One way to do it would be to have a `Transaction SpecialRemote cache`,
where SpecialRemote contains methods for Storer, Retriever, Remover, and
CheckPresent, that all expect to be passed a `cache`.
2014-08-06 18:28:36 +00:00
|
|
|
(simplyPrepare $ remove hooktype)
|
|
|
|
(simplyPrepare $ checkKey r hooktype)
|
2014-12-16 19:26:13 +00:00
|
|
|
Remote
|
|
|
|
{ uuid = u
|
|
|
|
, cost = cst
|
|
|
|
, name = Git.repoDescribe r
|
|
|
|
, storeKey = storeKeyDummy
|
|
|
|
, retrieveKeyFile = retreiveKeyFileDummy
|
|
|
|
, retrieveKeyFileCheap = retrieveCheap hooktype
|
|
|
|
, removeKey = removeKeyDummy
|
|
|
|
, checkPresent = checkPresentDummy
|
|
|
|
, checkPresentCheap = False
|
|
|
|
, whereisKey = Nothing
|
|
|
|
, remoteFsck = Nothing
|
|
|
|
, repairRepo = Nothing
|
|
|
|
, config = c
|
|
|
|
, localpath = Nothing
|
|
|
|
, repo = r
|
|
|
|
, gitconfig = gc
|
|
|
|
, readonly = False
|
|
|
|
, availability = GloballyAvailable
|
|
|
|
, remotetype = remote
|
|
|
|
, mkUnavailable = gen r u c $
|
|
|
|
gc { remoteAnnexHookType = Just "!dne!" }
|
|
|
|
, getInfo = return [("hooktype", hooktype)]
|
|
|
|
, claimUrl = Nothing
|
|
|
|
, checkUrl = Nothing
|
|
|
|
}
|
2013-01-01 17:52:47 +00:00
|
|
|
where
|
external special remotes mostly implemented (untested)
This has not been tested at all. It compiles!
The only known missing things are support for encryption, and for get/set
of special remote configuration, and of key state. (The latter needs
separate work to add a new per-key log file to store that state.)
Only thing I don't much like is that initremote needs to be passed both
type=external and externaltype=foo. It would be better to have just
type=foo
Most of this is quite straightforward code, that largely wrote itself given
the types. The only tricky parts were:
* Need to lock the remote when using it to eg make a request, because
in theory git-annex could have multiple threads that each try to use
a remote at the same time. I don't think that git-annex ever does
that currently, but better safe than sorry.
* Rather than starting up every external special remote program when
git-annex starts, they are started only on demand, when first used.
This will avoid slowdown, especially when running fast git-annex query
commands. Once started, they keep running until git-annex stops, currently,
which may not be ideal, but it's hard to know a better time to stop them.
* Bit of a chicken and egg problem with caching the cost of the remote,
because setting annex-cost in the git config needs the remote to already
be set up. Managed to finesse that.
This commit was sponsored by Lukas Anzinger.
2013-12-26 22:23:13 +00:00
|
|
|
hooktype = fromMaybe (error "missing hooktype") $ remoteAnnexHookType gc
|
2011-04-28 21:21:45 +00:00
|
|
|
|
2014-02-11 18:06:50 +00:00
|
|
|
hookSetup :: Maybe UUID -> Maybe CredPair -> RemoteConfig -> Annex (RemoteConfig, UUID)
|
|
|
|
hookSetup mu _ c = do
|
2013-09-07 22:38:00 +00:00
|
|
|
u <- maybe (liftIO genUUID) return mu
|
2011-07-15 16:47:14 +00:00
|
|
|
let hooktype = fromMaybe (error "Specify hooktype=") $
|
2011-05-15 06:49:43 +00:00
|
|
|
M.lookup "hooktype" c
|
glacier, S3: Fix bug that caused embedded creds to not be encypted using the remote's key.
encryptionSetup must be called before setRemoteCredPair. Otherwise,
the RemoteConfig doesn't have the cipher in it, and so no cipher is used to
encrypt the embedded creds.
This is a security fix for non-shared encryption methods!
For encryption=shared, there's no security problem, just an
inconsistentency in whether the embedded creds are encrypted.
This is very important to get right, so used some types to help ensure that
setRemoteCredPair is only run after encryptionSetup. Note that the external
special remote bypasses the type safety, since creds can be set after the
initial remote config, if the external special remote program requests it.
Also note that IA remotes never use encryption, so encryptionSetup is not
run for them at all, and again the type safety is bypassed.
This leaves two open questions:
1. What to do about S3 and glacier remotes that were set up
using encryption=pubkey/hybrid with embedcreds?
Such a git repo has a security hole embedded in it, and this needs to be
communicated to the user. Is the changelog enough?
2. enableremote won't work in such a repo, because git-annex will
try to decrypt the embedded creds, which are not encrypted, so fails.
This needs to be dealt with, especially for ecryption=shared repos,
which are not really broken, just inconsistently configured.
Noticing that problem for encryption=shared is what led to commit
fbdeeeed5fa276d94be587c8916d725eddcaf546, which tried to
fix the problem by not decrypting the embedded creds.
This commit was sponsored by Josh Taylor.
2014-09-18 21:07:17 +00:00
|
|
|
(c', _encsetup) <- encryptionSetup c
|
2011-04-28 21:21:45 +00:00
|
|
|
gitConfigSpecialRemote u c' "hooktype" hooktype
|
2013-09-07 22:38:00 +00:00
|
|
|
return (c', u)
|
2011-04-28 21:21:45 +00:00
|
|
|
|
2013-05-21 23:19:03 +00:00
|
|
|
hookEnv :: Action -> Key -> Maybe FilePath -> IO (Maybe [(String, String)])
|
|
|
|
hookEnv action k f = Just <$> mergeenv (fileenv f ++ keyenv)
|
2012-11-11 04:51:07 +00:00
|
|
|
where
|
2014-01-14 20:42:10 +00:00
|
|
|
mergeenv l = addEntries l <$> getEnvironment
|
2014-06-10 23:20:14 +00:00
|
|
|
envvar s v = ("ANNEX_" ++ s, v)
|
2012-11-11 04:51:07 +00:00
|
|
|
keyenv = catMaybes
|
2014-06-10 23:20:14 +00:00
|
|
|
[ Just $ envvar "KEY" (key2file k)
|
|
|
|
, Just $ envvar "ACTION" action
|
|
|
|
, envvar "HASH_1" <$> headMaybe hashbits
|
|
|
|
, envvar "HASH_2" <$> headMaybe (drop 1 hashbits)
|
2012-11-11 04:51:07 +00:00
|
|
|
]
|
|
|
|
fileenv Nothing = []
|
2014-06-10 23:20:14 +00:00
|
|
|
fileenv (Just file) = [envvar "FILE" file]
|
2012-11-11 04:51:07 +00:00
|
|
|
hashbits = map takeDirectory $ splitPath $ hashDirMixed k
|
2011-04-28 21:21:45 +00:00
|
|
|
|
2013-05-21 23:19:03 +00:00
|
|
|
lookupHook :: HookName -> Action -> Annex (Maybe String)
|
|
|
|
lookupHook hookname action = do
|
|
|
|
command <- getConfig (annexConfig hook) ""
|
2011-04-28 21:21:45 +00:00
|
|
|
if null command
|
|
|
|
then do
|
2013-09-26 03:19:01 +00:00
|
|
|
fallback <- getConfig (annexConfig hookfallback) ""
|
2013-05-21 23:19:03 +00:00
|
|
|
if null fallback
|
|
|
|
then do
|
|
|
|
warning $ "missing configuration for " ++ hook ++ " or " ++ hookfallback
|
|
|
|
return Nothing
|
|
|
|
else return $ Just fallback
|
2011-04-28 21:21:45 +00:00
|
|
|
else return $ Just command
|
2012-11-11 04:51:07 +00:00
|
|
|
where
|
2013-05-21 23:19:03 +00:00
|
|
|
hook = hookname ++ "-" ++ action ++ "-hook"
|
|
|
|
hookfallback = hookname ++ "-hook"
|
2011-04-28 21:21:45 +00:00
|
|
|
|
2013-05-21 23:19:03 +00:00
|
|
|
runHook :: HookName -> Action -> Key -> Maybe FilePath -> Annex Bool -> Annex Bool
|
|
|
|
runHook hook action k f a = maybe (return False) run =<< lookupHook hook action
|
2012-11-11 04:51:07 +00:00
|
|
|
where
|
|
|
|
run command = do
|
|
|
|
showOutput -- make way for hook output
|
2013-05-21 23:19:03 +00:00
|
|
|
ifM (liftIO $ boolSystemEnv "sh" [Param "-c", Param command] =<< hookEnv action k f)
|
2012-11-11 04:51:07 +00:00
|
|
|
( a
|
|
|
|
, do
|
|
|
|
warning $ hook ++ " hook exited nonzero!"
|
|
|
|
return False
|
|
|
|
)
|
2011-04-28 21:21:45 +00:00
|
|
|
|
2014-08-02 21:25:16 +00:00
|
|
|
store :: HookName -> Storer
|
|
|
|
store h = fileStorer $ \k src _p ->
|
2011-11-08 19:34:10 +00:00
|
|
|
runHook h "store" k (Just src) $ return True
|
2011-04-28 21:21:45 +00:00
|
|
|
|
2014-08-02 21:25:16 +00:00
|
|
|
retrieve :: HookName -> Retriever
|
|
|
|
retrieve h = fileRetriever $ \d k _p ->
|
|
|
|
unlessM (runHook h "retrieve" k (Just d) $ return True) $
|
|
|
|
error "failed to retrieve content"
|
2012-01-20 17:23:11 +00:00
|
|
|
|
2013-05-21 23:19:03 +00:00
|
|
|
retrieveCheap :: HookName -> Key -> FilePath -> Annex Bool
|
2012-01-20 17:23:11 +00:00
|
|
|
retrieveCheap _ _ _ = return False
|
2011-04-28 21:21:45 +00:00
|
|
|
|
run Preparer to get Remover and CheckPresent actions
This will allow special remotes to eg, open a http connection and reuse it,
while checking if chunks are present, or removing chunks.
S3 and WebDAV both need this to support chunks with reasonable speed.
Note that a special remote might want to cache a http connection across
multiple requests. A simple case of this is that CheckPresent is typically
called before Store or Remove. A remote using this interface can certianly
use a Preparer that eg, uses a MVar to cache a http connection.
However, it's up to the remote to then deal with things like stale or
stalled http connections when eg, doing a series of downloads from a remote
and other places. There could be long delays between calls to a remote,
which could lead to eg, http connection stalls; the machine might even
move to a new network, etc.
It might be nice to improve this interface later to allow
the simple case without needing to handle the full complex case.
One way to do it would be to have a `Transaction SpecialRemote cache`,
where SpecialRemote contains methods for Storer, Retriever, Remover, and
CheckPresent, that all expect to be passed a `cache`.
2014-08-06 18:28:36 +00:00
|
|
|
remove :: HookName -> Remover
|
2011-07-15 16:47:14 +00:00
|
|
|
remove h k = runHook h "remove" k Nothing $ return True
|
2011-04-28 21:21:45 +00:00
|
|
|
|
run Preparer to get Remover and CheckPresent actions
This will allow special remotes to eg, open a http connection and reuse it,
while checking if chunks are present, or removing chunks.
S3 and WebDAV both need this to support chunks with reasonable speed.
Note that a special remote might want to cache a http connection across
multiple requests. A simple case of this is that CheckPresent is typically
called before Store or Remove. A remote using this interface can certianly
use a Preparer that eg, uses a MVar to cache a http connection.
However, it's up to the remote to then deal with things like stale or
stalled http connections when eg, doing a series of downloads from a remote
and other places. There could be long delays between calls to a remote,
which could lead to eg, http connection stalls; the machine might even
move to a new network, etc.
It might be nice to improve this interface later to allow
the simple case without needing to handle the full complex case.
One way to do it would be to have a `Transaction SpecialRemote cache`,
where SpecialRemote contains methods for Storer, Retriever, Remover, and
CheckPresent, that all expect to be passed a `cache`.
2014-08-06 18:28:36 +00:00
|
|
|
checkKey :: Git.Repo -> HookName -> CheckPresent
|
2014-08-06 17:45:19 +00:00
|
|
|
checkKey r h k = do
|
2011-07-19 18:07:23 +00:00
|
|
|
showAction $ "checking " ++ Git.repoDescribe r
|
2013-05-21 23:19:03 +00:00
|
|
|
v <- lookupHook h action
|
2014-08-06 17:45:19 +00:00
|
|
|
liftIO $ check v
|
2012-11-11 04:51:07 +00:00
|
|
|
where
|
2014-10-09 18:53:13 +00:00
|
|
|
action = "checkpresent"
|
2012-11-11 04:51:07 +00:00
|
|
|
findkey s = key2file k `elem` lines s
|
2013-05-21 23:19:03 +00:00
|
|
|
check Nothing = error $ action ++ " hook misconfigured"
|
2012-11-11 04:51:07 +00:00
|
|
|
check (Just hook) = do
|
2014-06-10 23:20:14 +00:00
|
|
|
environ <- hookEnv action k Nothing
|
|
|
|
findkey <$> readProcessEnv "sh" ["-c", hook] environ
|