incremental verification for VURL

Sponsored-by: Brett Eisenberg on Patreon
This commit is contained in:
Joey Hess 2024-03-01 13:31:43 -04:00
parent 62e4c9d3b8
commit 4046f17ca0
No known key found for this signature in database
GPG key ID: DB12DB0FF05F8F38
2 changed files with 38 additions and 12 deletions

View file

@ -15,6 +15,8 @@ import Types.Key
import Types.Backend
import Logs.EquivilantKeys
import Backend.Variety
import Backend.Hash (descChecksum)
import Utility.Hash
backends :: [Backend]
backends = [backendVURL]
@ -30,14 +32,42 @@ backendVURL = Backend
-- because downloading the content from the web in
-- the first place records one.
[] -> return False
l -> do
eks -> do
let check ek = getbackend ek >>= \case
Nothing -> pure False
Just b -> case verifyKeyContent b of
Just verify -> verify ek f
Nothing -> pure False
anyM check l
, verifyKeyContentIncrementally = Nothing -- TODO
anyM check eks
, verifyKeyContentIncrementally = Just $ \k -> do
-- Run incremental verifiers for each equivilant key together,
-- and see if any of them succeed.
eks <- equivkeys k
let get = \ek -> getbackend ek >>= \case
Nothing -> pure Nothing
Just b -> case verifyKeyContentIncrementally b of
Nothing -> pure Nothing
Just va -> Just <$> va ek
l <- catMaybes <$> forM eks get
return $ IncrementalVerifier
{ updateIncrementalVerifier = \s ->
forM_ l $ flip updateIncrementalVerifier s
-- If there are no equivilant keys recorded somehow,
-- or if none of them support incremental verification,
-- this will return Nothing, which indicates that
-- incremental verification was not able to be
-- performed.
, finalizeIncrementalVerifier = do
r <- forM l finalizeIncrementalVerifier
return $ case catMaybes r of
[] -> Nothing
r' -> Just (or r')
, unableIncrementalVerifier =
forM_ l unableIncrementalVerifier
, positionIncrementalVerifier =
getM positionIncrementalVerifier l
, descIncrementalVerifier = descChecksum
}
, canUpgradeKey = Nothing
, fastMigrate = Nothing
-- Even if a hash is recorded on initial download from the web and

View file

@ -11,16 +11,12 @@ verify the content.
The web special remote can hash the content as it's downloading it from the
web, and record the resulting hash-based key.
> Status: This is implemented and working, but verifyKeyContentIncrementally
> needs to be implemented. Until it is, VURLs will not be as efficient
> as they could be.
> Status: Working, but `git-annex addurl --relaxed --verifiable` followed
> by `git-annex get` currently does 2 checksums in the get stage; it should
> only do one.
>
> Also `git-annex addurl --relaxed --verifiable` followed by `git-annex get`
> currently does 2 checksums in the get stage; it should only do one.
> Re-check this after implementing incremental verification.
>
> It's not yet possible to migrate an URL key to a VURL key. Should
> be easy to add support for this. --[[Joey]]
> It's not yet possible to migrate an URL key to a VURL key. Should be easy
> to add support for this. --[[Joey]]
## handling upgrades