git-annex/Remote/Hook.hs
Joey Hess c20358b671
incremental verify for byteRetriever special remotes
Several special remotes verify content while it is being retrieved,
avoiding a separate checksum pass. They are: S3, bup, ddar, and
gcrypt (with a local repository).

Not done when using chunking, yet.

Complicated by Retriever needing to change to be polymorphic. Which in turn
meant RankNTypes is needed, and also needed some code changes. The
change in Remote.External does not change behavior at all but avoids
the type checking failing because of a "rigid, skolem type" which
"would escape its scope". So I refactored slightly to make the type
checker's job easier there.

Unfortunately, directory uses fileRetriever (except when chunked),
so it is not amoung the improved ones. Fixing that would need a way for
FileRetriever to return a Verification. But, since the file retrieved
may be encrypted or chunked, it would be extra work to always
incrementally checksum the file while retrieving it. Hm.

Some other special remotes use fileRetriever, and so don't get incremental
verification, but could be converted to byteRetriever later. One is
GitLFS, which uses downloadConduit, which writes to the file, so could
verify as it goes. Other special remotes like web could too, but don't
use Remote.Helper.Special and so will need to be addressed separately.

Sponsored-by: Dartmouth College's DANDI project
2021-08-11 14:20:38 -04:00

183 lines
5.6 KiB
Haskell

{- A remote that provides hooks to run shell commands.
-
- Copyright 2011-2020 Joey Hess <id@joeyh.name>
-
- Licensed under the GNU AGPL version 3 or higher.
-}
{-# LANGUAGE RankNTypes #-}
module Remote.Hook (remote) where
import Annex.Common
import Types.Remote
import Types.Creds
import qualified Git
import Git.Types (fromConfigKey, fromConfigValue)
import Config
import Config.Cost
import Annex.UUID
import Annex.SpecialRemote.Config
import Remote.Helper.Special
import Remote.Helper.ExportImport
import Utility.Env
import Messages.Progress
import Types.ProposedAccepted
import qualified Data.Map as M
type Action = String
type HookName = String
remote :: RemoteType
remote = specialRemoteType $ RemoteType
{ typename = "hook"
, enumerate = const (findSpecialRemotes "hooktype")
, generate = gen
, configParser = mkRemoteConfigParser
[ optionalStringParser hooktypeField
(FieldDesc "(required) specify collection of hooks to use")
]
, setup = hookSetup
, exportSupported = exportUnsupported
, importSupported = importUnsupported
, thirdPartyPopulated = False
}
hooktypeField :: RemoteConfigField
hooktypeField = Accepted "hooktype"
gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> RemoteStateHandle -> Annex (Maybe Remote)
gen r u rc gc rs = do
c <- parsedRemoteConfig remote rc
cst <- remoteCost gc expensiveRemoteCost
return $ Just $ specialRemote c
(store hooktype)
(retrieve hooktype)
(remove hooktype)
(checkKey hooktype)
Remote
{ uuid = u
, cost = cst
, name = Git.repoDescribe r
, storeKey = storeKeyDummy
, retrieveKeyFile = retrieveKeyFileDummy
, retrieveKeyFileCheap = Nothing
-- A hook could use http and be vulnerable to
-- redirect to file:// attacks, etc.
, retrievalSecurityPolicy = mkRetrievalVerifiableKeysSecure gc
, removeKey = removeKeyDummy
, lockContent = Nothing
, checkPresent = checkPresentDummy
, checkPresentCheap = False
, exportActions = exportUnsupported
, importActions = importUnsupported
, whereisKey = Nothing
, remoteFsck = Nothing
, repairRepo = Nothing
, config = c
, localpath = Nothing
, getRepo = return r
, gitconfig = gc
, readonly = False
, appendonly = False
, untrustworthy = False
, availability = GloballyAvailable
, remotetype = remote
, mkUnavailable = gen r u rc
(gc { remoteAnnexHookType = Just "!dne!" })
rs
, getInfo = return [("hooktype", hooktype)]
, claimUrl = Nothing
, checkUrl = Nothing
, remoteStateHandle = rs
}
where
hooktype = fromMaybe (giveup "missing hooktype") $ remoteAnnexHookType gc
hookSetup :: SetupStage -> Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID)
hookSetup _ mu _ c gc = do
u <- maybe (liftIO genUUID) return mu
let hooktype = maybe (giveup "Specify hooktype=") fromProposedAccepted $
M.lookup hooktypeField c
(c', _encsetup) <- encryptionSetup c gc
gitConfigSpecialRemote u c' [("hooktype", hooktype)]
return (c', u)
hookEnv :: Action -> Key -> Maybe FilePath -> IO (Maybe [(String, String)])
hookEnv action k f = Just <$> mergeenv (fileenv f ++ keyenv)
where
mergeenv l = addEntries l <$> getEnvironment
envvar s v = ("ANNEX_" ++ s, v)
keyenv = catMaybes
[ Just $ envvar "KEY" (serializeKey k)
, Just $ envvar "ACTION" action
, envvar "HASH_1" <$> headMaybe hashbits
, envvar "HASH_2" <$> headMaybe (drop 1 hashbits)
]
fileenv Nothing = []
fileenv (Just file) = [envvar "FILE" file]
hashbits = map takeDirectory $ splitPath $
fromRawFilePath $ hashDirMixed def k
lookupHook :: HookName -> Action -> Annex (Maybe String)
lookupHook hookname action = do
command <- fromConfigValue <$> getConfig hook mempty
if null command
then do
fallback <- fromConfigValue <$> getConfig hookfallback mempty
if null fallback
then do
warning $ "missing configuration for " ++ fromConfigKey hook ++ " or " ++ fromConfigKey hookfallback
return Nothing
else return $ Just fallback
else return $ Just command
where
hook = annexConfig $ encodeBS $ hookname ++ "-" ++ action ++ "-hook"
hookfallback = annexConfig $ encodeBS $ hookname ++ "-hook"
runHook :: HookName -> Action -> Key -> Maybe FilePath -> Annex ()
runHook hook action k f = lookupHook hook action >>= \case
Just command -> do
showOutput -- make way for hook output
environ <- liftIO (hookEnv action k f)
unlessM (progressCommandEnv "sh" [Param "-c", Param command] environ) $
giveup $ hook ++ " hook exited nonzero!"
Nothing -> giveup $ action ++ " hook misconfigured"
runHook' :: HookName -> Action -> Key -> Maybe FilePath -> Annex Bool -> Annex Bool
runHook' hook action k f a = maybe (return False) run =<< lookupHook hook action
where
run command = do
showOutput -- make way for hook output
ifM (progressCommandEnv "sh" [Param "-c", Param command] =<< liftIO (hookEnv action k f))
( a
, do
warning $ hook ++ " hook exited nonzero!"
return False
)
store :: HookName -> Storer
store h = fileStorer $ \k src _p -> runHook h "store" k (Just src)
retrieve :: HookName -> Retriever
retrieve h = fileRetriever $ \d k _p ->
unlessM (runHook' h "retrieve" k (Just d) $ return True) $
giveup "failed to retrieve content"
remove :: HookName -> Remover
remove h k =
unlessM (runHook' h "remove" k Nothing $ return True) $
giveup "failed to remove content"
checkKey :: HookName -> CheckPresent
checkKey h k = do
v <- lookupHook h action
liftIO $ check v
where
action = "checkpresent"
findkey s = serializeKey k `elem` lines s
check Nothing = giveup $ action ++ " hook misconfigured"
check (Just hook) = do
environ <- hookEnv action k Nothing
findkey <$> readProcessEnv "sh" ["-c", hook] environ