incremental verification for web special remote
Except when configuration makes curl be used. It did not seem worth
trying to tail the file when curl is downloading.
But when an interrupted download is resumed, it does not read the whole
existing file to hash it. Same reason discussed in
commit 7eb3742e4b
; that could take a long
time with no progress being displayed. And also there's an open http
request, which needs to be consumed; taking a long time to hash the file
might cause it to time out.
Also in passing implemented it for git and external special remotes when
downloading from the web. Several others like S3 are within striking
distance now as well.
Sponsored-by: Dartmouth College's DANDI project
This commit is contained in:
parent
88b63a43fa
commit
d154e7022e
15 changed files with 101 additions and 67 deletions
|
@ -1,6 +1,6 @@
|
|||
{- Web remote.
|
||||
-
|
||||
- Copyright 2011 Joey Hess <id@joeyh.name>
|
||||
- Copyright 2011-2021 Joey Hess <id@joeyh.name>
|
||||
-
|
||||
- Licensed under the GNU AGPL version 3 or higher.
|
||||
-}
|
||||
|
@ -13,6 +13,7 @@ import Remote.Helper.ExportImport
|
|||
import qualified Git
|
||||
import qualified Git.Construct
|
||||
import Annex.Content
|
||||
import Annex.Verify
|
||||
import Config.Cost
|
||||
import Config
|
||||
import Logs.Web
|
||||
|
@ -82,19 +83,29 @@ gen r _ rc gc rs = do
|
|||
}
|
||||
|
||||
downloadKey :: Key -> AssociatedFile -> FilePath -> MeterUpdate -> VerifyConfig -> Annex Verification
|
||||
downloadKey key _af dest p _ = do
|
||||
get =<< getWebUrls key
|
||||
return UnVerified
|
||||
downloadKey key _af dest p vc = go =<< getWebUrls key
|
||||
where
|
||||
get [] = giveup "no known url"
|
||||
get urls = do
|
||||
r <- untilTrue urls $ \u -> do
|
||||
let (u', downloader) = getDownloader u
|
||||
case downloader of
|
||||
YoutubeDownloader -> youtubeDlTo key u' dest p
|
||||
_ -> Url.withUrlOptions $ downloadUrl key p [u'] dest
|
||||
unless r $
|
||||
giveup "download failed"
|
||||
go [] = giveup "no known url"
|
||||
go urls = getM dl urls >>= \case
|
||||
Just v -> return v
|
||||
Nothing -> giveup "download failed"
|
||||
|
||||
dl u = do
|
||||
let (u', downloader) = getDownloader u
|
||||
case downloader of
|
||||
YoutubeDownloader ->
|
||||
ifM (youtubeDlTo key u' dest p)
|
||||
( return (Just UnVerified)
|
||||
, return Nothing
|
||||
)
|
||||
_ -> do
|
||||
iv <- startVerifyKeyContentIncrementally vc key
|
||||
ifM (Url.withUrlOptions $ downloadUrl key p iv [u'] dest)
|
||||
( finishVerifyKeyContentIncrementally iv >>= \case
|
||||
(True, v) -> return (Just v)
|
||||
(False, _) -> return Nothing
|
||||
, return Nothing
|
||||
)
|
||||
|
||||
uploadKey :: Key -> AssociatedFile -> MeterUpdate -> Annex ()
|
||||
uploadKey _ _ _ = giveup "upload to web not supported"
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue