mirrors/git-annex
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

incremental verification for retrieval from all export remotes

Browse source 
Only for export remotes so far, not export/import.

Sponsored-by: Dartmouth College's Datalad project
This commit is contained in:
Joey Hess 2022-05-09 13:18:47 -04:00
parent 90950a37e5
commit 2f2701137d
No known key found for this signature in database
GPG key ID: DB12DB0FF05F8F38
11 changed files with 73 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

9
Remote/S3.hs
View file

@ -60,13 +60,13 @@ import Types.MetaData
import Types.ProposedAccepted
import Types.NumCopies
import Utility.Metered
import Utility.Hash (IncrementalVerifier)
import Utility.DataUnits
import Annex.Content
import qualified Annex.Url as Url
import Utility.Url (extractFromResourceT)
import Annex.Url (getUrlOptions, withUrlOptions, UrlOptions(..))
import Utility.Env
import Annex.Verify
type BucketName = String
type BucketObject = String
@ -496,15 +496,14 @@ storeExportS3' hv r rs info magic f k loc p = withS3Handle hv $ \case
return (metag, mvid)
retrieveExportS3 :: S3HandleVar -> Remote -> S3Info -> Key -> ExportLocation -> FilePath -> MeterUpdate -> Annex Verification
retrieveExportS3 hv r info _k loc f p = do
retrieveExportS3 hv r info k loc f p = verifyKeyContentIncrementally AlwaysVerify k $ \iv ->
withS3Handle hv $ \case
Just h -> retrieveHelper info h (Left (T.pack exportloc)) f p Nothing
Just h -> retrieveHelper info h (Left (T.pack exportloc)) f p iv
Nothing -> case getPublicUrlMaker info of
Just geturl -> either giveup return =<<
Url.withUrlOptions
(Url.download' p Nothing (geturl exportloc) f)
(Url.download' p iv (geturl exportloc) f)
Nothing -> giveup $ needS3Creds (uuid r)
return UnVerified
where
exportloc = bucketExportLocation info loc