1243af4a18
Added Maybe POSIXTime to SafeDropProof, which gets set when the proof is based on a LockedCopy. If there are several LockedCopies, it uses the closest expiry time. That is not optimal, it may be that the proof expires based on one LockedCopy but another one has not expired. But that seems unlikely to really happen, and anyway the user can just re-run a drop if it fails due to expiry. Pass the SafeDropProof to removeKey, which is responsible for checking it for expiry in situations where that could be a problem. Which really only means in Remote.Git. Made Remote.Git check expiry when dropping from a local remote. Checking expiry when dropping from a P2P remote is not yet implemented. P2P.Protocol.remove has SafeDropProof plumbed through to it for that purpose. Fixing the remaining 2 build warnings should complete this work. Note that the use of a POSIXTime here means that if the clock gets set forward while git-annex is in the middle of a drop, it may say that dropping took too long. That seems ok. Less ok is that if the clock gets turned back a sufficient amount (eg 5 minutes), proof expiry won't be noticed. It might be better to use the Monotonic clock, but that doesn't advance when a laptop is suspended, and while there is the linux Boottime clock, that is not available on other systems. Perhaps a combination of POSIXTime and the Monotonic clock could detect laptop suspension and also detect clock being turned back? There is a potential future flag day where p2pDefaultLockContentRetentionDuration is not assumed, but is probed using the P2P protocol, and peers that don't support it can no longer produce a LockedCopy. Until that happens, when git-annex is communicating with older peers there is a risk of data loss when a ssh connection closes during LOCKCONTENT.
367 lines
11 KiB
Haskell
367 lines
11 KiB
Haskell
{- Amazon Glacier remotes.
|
|
-
|
|
- Copyright 2012-2021 Joey Hess <id@joeyh.name>
|
|
-
|
|
- Licensed under the GNU AGPL version 3 or higher.
|
|
-}
|
|
|
|
{-# LANGUAGE RankNTypes #-}
|
|
|
|
module Remote.Glacier (remote, jobList, checkSaneGlacierCommand) where
|
|
|
|
import qualified Data.Map as M
|
|
import qualified Data.Text as T
|
|
import qualified Data.ByteString as S
|
|
import qualified Data.ByteString.Lazy as L
|
|
|
|
import Annex.Common
|
|
import Types.Remote
|
|
import qualified Git
|
|
import Config
|
|
import Config.Cost
|
|
import Annex.SpecialRemote.Config
|
|
import Remote.Helper.Special
|
|
import Remote.Helper.ExportImport
|
|
import qualified Remote.Helper.AWS as AWS
|
|
import Creds
|
|
import Utility.Metered
|
|
import Annex.UUID
|
|
import Utility.Env
|
|
import Types.ProposedAccepted
|
|
import Utility.Hash (IncrementalVerifier)
|
|
|
|
type Vault = String
|
|
type Archive = FilePath
|
|
|
|
remote :: RemoteType
|
|
remote = specialRemoteType $ RemoteType
|
|
{ typename = "glacier"
|
|
, enumerate = const (findSpecialRemotes "glacier")
|
|
, generate = gen
|
|
, configParser = mkRemoteConfigParser
|
|
[ optionalStringParser datacenterField
|
|
(FieldDesc "S3 datacenter to use")
|
|
, optionalStringParser vaultField
|
|
(FieldDesc "name to use for vault")
|
|
, optionalStringParser fileprefixField
|
|
(FieldDesc "prefix to add to filenames in the vault")
|
|
, optionalStringParser AWS.s3credsField HiddenField
|
|
]
|
|
, setup = glacierSetup
|
|
, exportSupported = exportUnsupported
|
|
, importSupported = importUnsupported
|
|
, thirdPartyPopulated = False
|
|
}
|
|
|
|
datacenterField :: RemoteConfigField
|
|
datacenterField = Accepted "datacenter"
|
|
|
|
vaultField :: RemoteConfigField
|
|
vaultField = Accepted "vault"
|
|
|
|
fileprefixField :: RemoteConfigField
|
|
fileprefixField = Accepted "fileprefix"
|
|
|
|
gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> RemoteStateHandle -> Annex (Maybe Remote)
|
|
gen r u rc gc rs = do
|
|
c <- parsedRemoteConfig remote rc
|
|
cst <- remoteCost gc c veryExpensiveRemoteCost
|
|
return (new c cst)
|
|
where
|
|
new c cst = Just $ specialRemote' specialcfg c
|
|
(store this)
|
|
(retrieve this)
|
|
(remove this)
|
|
(checkKey this)
|
|
this
|
|
where
|
|
this = Remote
|
|
{ uuid = u
|
|
, cost = cst
|
|
, name = Git.repoDescribe r
|
|
, storeKey = storeKeyDummy
|
|
, retrieveKeyFile = retrieveKeyFileDummy
|
|
, retrieveKeyFileCheap = Nothing
|
|
-- glacier-cli does not follow redirects and does
|
|
-- not support file://, as far as we know, but
|
|
-- there's no guarantee that will continue to be
|
|
-- the case, so require verifiable keys.
|
|
, retrievalSecurityPolicy = mkRetrievalVerifiableKeysSecure gc
|
|
, removeKey = removeKeyDummy
|
|
, lockContent = Nothing
|
|
, checkPresent = checkPresentDummy
|
|
, checkPresentCheap = False
|
|
, exportActions = exportUnsupported
|
|
, importActions = importUnsupported
|
|
, whereisKey = Nothing
|
|
, remoteFsck = Nothing
|
|
, repairRepo = Nothing
|
|
, config = c
|
|
, getRepo = return r
|
|
, gitconfig = gc
|
|
, localpath = Nothing
|
|
, readonly = False
|
|
, appendonly = False
|
|
, untrustworthy = False
|
|
, availability = pure GloballyAvailable
|
|
, remotetype = remote
|
|
, mkUnavailable = return Nothing
|
|
, getInfo = includeCredsInfo c (AWS.creds u) $
|
|
[ ("glacier vault", getVault c) ]
|
|
, claimUrl = Nothing
|
|
, checkUrl = Nothing
|
|
, remoteStateHandle = rs
|
|
}
|
|
specialcfg = (specialRemoteCfg c)
|
|
-- Disabled until jobList gets support for chunks.
|
|
{ chunkConfig = NoChunks
|
|
}
|
|
|
|
glacierSetup :: SetupStage -> Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID)
|
|
glacierSetup ss mu mcreds c gc = do
|
|
u <- maybe (liftIO genUUID) return mu
|
|
glacierSetup' ss u mcreds c gc
|
|
glacierSetup' :: SetupStage -> UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID)
|
|
glacierSetup' ss u mcreds c gc = do
|
|
(c', encsetup) <- encryptionSetup (c `M.union` defaults) gc
|
|
pc <- either giveup return . parseRemoteConfig c'
|
|
=<< configParser remote c'
|
|
c'' <- setRemoteCredPair ss encsetup pc gc (AWS.creds u) mcreds
|
|
pc' <- either giveup return . parseRemoteConfig c''
|
|
=<< configParser remote c''
|
|
case ss of
|
|
Init -> genVault pc' gc u
|
|
_ -> return ()
|
|
gitConfigSpecialRemote u c'' [("glacier", "true")]
|
|
return (c'', u)
|
|
where
|
|
remotename = fromJust (lookupName c)
|
|
defvault = remotename ++ "-" ++ fromUUID u
|
|
defaults = M.fromList
|
|
[ (datacenterField, Proposed $ T.unpack $ AWS.defaultRegion AWS.Glacier)
|
|
, (vaultField, Proposed defvault)
|
|
]
|
|
|
|
store :: Remote -> Storer
|
|
store r k b p = do
|
|
checkNonEmpty k
|
|
byteStorer (store' r) k b p
|
|
|
|
checkNonEmpty :: Key -> Annex ()
|
|
checkNonEmpty k
|
|
| fromKey keySize k == Just 0 =
|
|
giveup "Cannot store empty files in Glacier."
|
|
| otherwise = return ()
|
|
|
|
store' :: Remote -> Key -> L.ByteString -> MeterUpdate -> Annex ()
|
|
store' r k b p = go =<< glacierEnv c gc u
|
|
where
|
|
c = config r
|
|
gc = gitconfig r
|
|
u = uuid r
|
|
params = glacierParams c
|
|
[ Param "archive"
|
|
, Param "upload"
|
|
, Param "--name", Param $ archive r k
|
|
, Param $ getVault $ config r
|
|
, Param "-"
|
|
]
|
|
go Nothing = giveup "Glacier not usable."
|
|
go (Just e) =
|
|
let cmd = (proc "glacier" (toCommand params)) { env = Just e }
|
|
{ std_in = CreatePipe }
|
|
in liftIO $ withCreateProcess cmd (go' cmd)
|
|
go' cmd (Just hin) _ _ pid = do
|
|
meteredWrite p (S.hPut hin) b
|
|
hClose hin
|
|
forceSuccessProcess cmd pid
|
|
go' _ _ _ _ _ = error "internal"
|
|
|
|
retrieve :: forall a. Remote -> Key -> MeterUpdate -> Maybe IncrementalVerifier -> (ContentSource -> Annex a) -> Annex a
|
|
retrieve = byteRetriever . retrieve'
|
|
|
|
retrieve' :: forall a. Remote -> Key -> (L.ByteString -> Annex a) -> Annex a
|
|
retrieve' r k sink = go =<< glacierEnv c gc u
|
|
where
|
|
c = config r
|
|
gc = gitconfig r
|
|
u = uuid r
|
|
params = glacierParams c
|
|
[ Param "archive"
|
|
, Param "retrieve"
|
|
, Param "-o-"
|
|
, Param $ getVault $ config r
|
|
, Param $ archive r k
|
|
]
|
|
go Nothing = giveup "cannot retrieve from glacier"
|
|
go (Just environ) = do
|
|
let p = (proc "glacier" (toCommand params))
|
|
{ env = Just environ
|
|
, std_out = CreatePipe
|
|
}
|
|
bracketIO (createProcess p) cleanupProcess (go' p)
|
|
go' p (_, Just h, _, pid) = do
|
|
let cleanup = liftIO $ do
|
|
hClose h
|
|
forceSuccessProcess p pid
|
|
flip finally cleanup $ do
|
|
-- Glacier cannot store empty files, so if
|
|
-- the output is empty, the content is not
|
|
-- available yet.
|
|
whenM (liftIO $ hIsEOF h) $
|
|
giveup "Content is not available from glacier yet. Recommend you wait up to 4 hours, and then run this command again."
|
|
sink =<< liftIO (L.hGetContents h)
|
|
go' _ _ = error "internal"
|
|
|
|
remove :: Remote -> Remover
|
|
remove r _proof k = unlessM go $
|
|
giveup "removal from glacier failed"
|
|
where
|
|
go = glacierAction r
|
|
[ Param "archive"
|
|
, Param "delete"
|
|
, Param $ getVault $ config r
|
|
, Param $ archive r k
|
|
]
|
|
|
|
checkKey :: Remote -> CheckPresent
|
|
checkKey r k = go =<< glacierEnv (config r) (gitconfig r) (uuid r)
|
|
where
|
|
go Nothing = giveup "cannot check glacier"
|
|
go (Just e) = do
|
|
{- glacier checkpresent outputs the archive name to stdout if
|
|
- it's present. -}
|
|
s <- liftIO $ readProcessEnv "glacier" (toCommand params) (Just e)
|
|
let probablypresent = serializeKey k `elem` lines s
|
|
if probablypresent
|
|
then giveup untrusted
|
|
else return False
|
|
|
|
params = glacierParams (config r)
|
|
[ Param "archive"
|
|
, Param "checkpresent"
|
|
, Param $ getVault $ config r
|
|
, Param "--quiet"
|
|
, Param $ archive r k
|
|
]
|
|
|
|
untrusted = unlines
|
|
[ "Glacier's inventory says it has a copy."
|
|
, "However, the inventory could be out of date, if it was recently removed."
|
|
]
|
|
|
|
glacierAction :: Remote -> [CommandParam] -> Annex Bool
|
|
glacierAction r = runGlacier (config r) (gitconfig r) (uuid r)
|
|
|
|
runGlacier :: ParsedRemoteConfig -> RemoteGitConfig -> UUID -> [CommandParam] -> Annex Bool
|
|
runGlacier c gc u params = go =<< glacierEnv c gc u
|
|
where
|
|
go Nothing = return False
|
|
go (Just e) = liftIO $
|
|
boolSystemEnv "glacier" (glacierParams c params) (Just e)
|
|
|
|
glacierParams :: ParsedRemoteConfig -> [CommandParam] -> [CommandParam]
|
|
glacierParams c params = datacenter:params
|
|
where
|
|
datacenter = Param $ "--region=" ++
|
|
fromMaybe (giveup "Missing datacenter configuration")
|
|
(getRemoteConfigValue datacenterField c)
|
|
|
|
glacierEnv :: ParsedRemoteConfig -> RemoteGitConfig -> UUID -> Annex (Maybe [(String, String)])
|
|
glacierEnv c gc u = do
|
|
liftIO checkSaneGlacierCommand
|
|
go =<< getRemoteCredPairFor "glacier" c gc creds
|
|
where
|
|
go Nothing = return Nothing
|
|
go (Just (user, pass)) = do
|
|
e <- liftIO getEnvironment
|
|
return $ Just $ addEntries [(uk, user), (pk, pass)] e
|
|
|
|
creds = AWS.creds u
|
|
(uk, pk) = credPairEnvironment creds
|
|
|
|
getVault :: ParsedRemoteConfig -> Vault
|
|
getVault = fromMaybe (giveup "Missing vault configuration")
|
|
. getRemoteConfigValue vaultField
|
|
|
|
archive :: Remote -> Key -> Archive
|
|
archive r k = fileprefix ++ serializeKey k
|
|
where
|
|
fileprefix = fromMaybe "" $
|
|
getRemoteConfigValue fileprefixField $ config r
|
|
|
|
genVault :: ParsedRemoteConfig -> RemoteGitConfig -> UUID -> Annex ()
|
|
genVault c gc u = unlessM (runGlacier c gc u params) $
|
|
giveup "Failed creating glacier vault."
|
|
where
|
|
params =
|
|
[ Param "vault"
|
|
, Param "create"
|
|
, Param $ getVault c
|
|
]
|
|
|
|
{- Partitions the input list of keys into ones which have
|
|
- glacier retieval jobs that have succeeded, or failed.
|
|
-
|
|
- A complication is that `glacier job list` will display the encrypted
|
|
- keys when the remote is encrypted.
|
|
-
|
|
- Dealing with encrypted chunked keys would be tricky. However, there
|
|
- seems to be no benefit to using chunking with glacier, so chunking is
|
|
- not supported.
|
|
-}
|
|
jobList :: Remote -> [Key] -> Annex ([Key], [Key])
|
|
jobList r keys = go =<< glacierEnv (config r) (gitconfig r) (uuid r)
|
|
where
|
|
params = [ Param "job", Param "list" ]
|
|
nada = ([], [])
|
|
myvault = getVault $ config r
|
|
|
|
go Nothing = return nada
|
|
go (Just e) = do
|
|
v <- liftIO $ catchMaybeIO $
|
|
readProcessEnv "glacier" (toCommand params) (Just e)
|
|
maybe (return nada) extract v
|
|
|
|
extract s = do
|
|
let result@(succeeded, failed) =
|
|
parse nada $ (map words . lines) s
|
|
if result == nada
|
|
then return nada
|
|
else do
|
|
enckeys <- forM keys $ \k ->
|
|
maybe k (\(_, enck) -> enck k)
|
|
<$> cipherKey (config r) (gitconfig r)
|
|
let keymap = M.fromList $ zip enckeys keys
|
|
let convert = mapMaybe (`M.lookup` keymap)
|
|
return (convert succeeded, convert failed)
|
|
|
|
parse c [] = c
|
|
parse c@(succeeded, failed) ((status:_date:vault:key:[]):rest)
|
|
| vault == myvault =
|
|
case deserializeKey key of
|
|
Nothing -> parse c rest
|
|
Just k
|
|
| "a/d" `isPrefixOf` status ->
|
|
parse (k:succeeded, failed) rest
|
|
| "a/e" `isPrefixOf` status ->
|
|
parse (succeeded, k:failed) rest
|
|
| otherwise ->
|
|
parse c rest
|
|
parse c (_:rest) = parse c rest
|
|
|
|
-- boto's version of glacier exits 0 when given a parameter it doesn't
|
|
-- understand. See https://github.com/boto/boto/issues/2942
|
|
checkSaneGlacierCommand :: IO ()
|
|
checkSaneGlacierCommand =
|
|
whenM ((Nothing /=) <$> catchMaybeIO shouldfail) $
|
|
giveup wrongcmd
|
|
where
|
|
test = proc "glacier" ["--compatibility-test-git-annex"]
|
|
shouldfail = withNullHandle $ \nullh ->
|
|
let p = test
|
|
{ std_out = UseHandle nullh
|
|
, std_err = UseHandle nullh
|
|
}
|
|
in withCreateProcess p $ \_ _ _ -> forceSuccessProcess p
|
|
wrongcmd = "The glacier program in PATH seems to be from boto, not glacier-cli. Cannot use this program."
|