incremental checksum on download from ssh or p2p
Checksum as content is received from a remote git-annex repository, rather than doing it in a second pass. Not tested at all yet, but I imagine it will work! Not implemented for any special remotes, and also not implemented for copies from local remotes. It may be that, for local remotes, it will suffice to use rsync, rely on its checksumming, and simply return Verified. (It would still make a checksumming pass when cp is used for COW, I guess.)
This commit is contained in:
parent
ed684f651e
commit
62e152f210
17 changed files with 118 additions and 52 deletions
|
@ -11,6 +11,7 @@ import Annex.Common
|
|||
import Remote.Helper.Chunked
|
||||
import Utility.Metered
|
||||
|
||||
import qualified Data.ByteString as S
|
||||
import qualified Data.ByteString.Lazy as L
|
||||
|
||||
{- This is an extension that's added to the usual file (or whatever)
|
||||
|
@ -117,4 +118,4 @@ meteredWriteFileChunks :: MeterUpdate -> FilePath -> [v] -> (v -> IO L.ByteStrin
|
|||
meteredWriteFileChunks meterupdate dest chunks feeder =
|
||||
withBinaryFile dest WriteMode $ \h ->
|
||||
forM_ chunks $
|
||||
meteredWrite meterupdate h <=< feeder
|
||||
meteredWrite meterupdate (S.hPut h) <=< feeder
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
{- Helpers for remotes using the git-annex P2P protocol.
|
||||
-
|
||||
- Copyright 2016-2020 Joey Hess <id@joeyh.name>
|
||||
- Copyright 2016-2021 Joey Hess <id@joeyh.name>
|
||||
-
|
||||
- Licensed under the GNU AGPL version 3 or higher.
|
||||
-}
|
||||
|
@ -17,6 +17,7 @@ import Annex.Content
|
|||
import Messages.Progress
|
||||
import Utility.Metered
|
||||
import Types.NumCopies
|
||||
import Backend
|
||||
|
||||
import Control.Concurrent
|
||||
|
||||
|
@ -39,10 +40,11 @@ store runner k af p = do
|
|||
Just False -> giveup "transfer failed"
|
||||
Nothing -> remoteUnavail
|
||||
|
||||
retrieve :: (MeterUpdate -> ProtoRunner (Bool, Verification)) -> Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex Verification
|
||||
retrieve runner k af dest p =
|
||||
retrieve :: VerifyConfig -> (MeterUpdate -> ProtoRunner (Bool, Verification)) -> Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex Verification
|
||||
retrieve verifyconfig runner k af dest p = do
|
||||
iv <- startVerifyKeyContentIncrementally verifyconfig k
|
||||
metered (Just p) k $ \m p' ->
|
||||
runner p' (P2P.get dest k af m p') >>= \case
|
||||
runner p' (P2P.get dest k iv af m p') >>= \case
|
||||
Just (True, v) -> return v
|
||||
Just (False, _) -> giveup "transfer failed"
|
||||
Nothing -> remoteUnavail
|
||||
|
|
|
@ -295,7 +295,7 @@ sink dest enc c mh mp content = case (enc, mh, content) of
|
|||
Just h -> liftIO $ b `streamto` h
|
||||
Nothing -> liftIO $ bracket opendest hClose (b `streamto`)
|
||||
streamto b h = case mp of
|
||||
Just p -> meteredWrite p h b
|
||||
Just p -> meteredWrite p (S.hPut h) b
|
||||
Nothing -> L.hPut h b
|
||||
opendest = openBinaryFile dest WriteMode
|
||||
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue