mirrors/git-annex
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

import: Retry downloads that fail

Browse source 
Also, using the transfer machinery for this makes eg, git-annex info show
in-progress imports, and makes --notify-start/finish work.
This commit is contained in:
Joey Hess 2020-09-04 13:49:57 -04:00
parent 46eb48d7c0
commit 2bb933eb60
No known key found for this signature in database
GPG key ID: DB12DB0FF05F8F38
3 changed files with 27 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

30
Annex/Import.hs
View file

@ -35,6 +35,7 @@ import Annex.Content
import Annex.Export import Annex.Export
import Annex.RemoteTrackingBranch import Annex.RemoteTrackingBranch
import Annex.HashObject import Annex.HashObject
import Annex.Transfer
import Command import Command
import Backend import Backend
import Types.Key import Types.Key
@ -411,8 +412,13 @@ importKeys remote importtreeconfig importcontent importablecontents = do
const runimport const runimport
dodownload cidmap db (loc, (cid, sz)) largematcher = do dodownload cidmap db (loc, (cid, sz)) largematcher = do
f <- locworktreefile loc
let af = AssociatedFile (Just f)
let downloader tmpfile p = do let downloader tmpfile p = do
k <- Remote.retrieveExportWithContentIdentifier ia loc cid tmpfile (mkkey loc tmpfile largematcher) p k <- Remote.retrieveExportWithContentIdentifier
ia loc cid tmpfile
(mkkey f tmpfile largematcher)
p
case keyGitSha k of case keyGitSha k of
Nothing -> do Nothing -> do
ok <- moveAnnex k tmpfile ok <- moveAnnex k tmpfile
@ -431,16 +437,17 @@ importKeys remote importtreeconfig importcontent importablecontents = do
warning (show e) warning (show e)
return Nothing return Nothing
checkDiskSpaceToGet tmpkey Nothing $ checkDiskSpaceToGet tmpkey Nothing $
withTmp tmpkey $ \tmpfile -> notifyTransfer Download af $
metered Nothing tmpkey $ download (Remote.uuid remote) tmpkey af stdRetry $ \p ->
const (rundownload tmpfile) withTmp tmpkey $ \tmpfile ->
metered (Just p) tmpkey $
const (rundownload tmpfile)
where where
tmpkey = importKey cid sz tmpkey = importKey cid sz
ia = Remote.importActions remote ia = Remote.importActions remote
mkkey loc tmpfile largematcher = do mkkey f tmpfile largematcher = do
f <- fromRepo $ fromTopFilePath $ locworktreefilename loc
matcher <- largematcher (fromRawFilePath f) matcher <- largematcher (fromRawFilePath f)
let mi = MatchingFile FileInfo let mi = MatchingFile FileInfo
{ matchFile = f { matchFile = f
@ -458,10 +465,11 @@ importKeys remote importtreeconfig importcontent importablecontents = do
fst <$> genKey ks nullMeterUpdate backend fst <$> genKey ks nullMeterUpdate backend
else gitShaKey <$> hashFile tmpfile else gitShaKey <$> hashFile tmpfile
locworktreefilename loc = asTopFilePath $ case importtreeconfig of locworktreefile loc = fromRepo $ fromTopFilePath $ asTopFilePath $
ImportTree -> fromImportLocation loc case importtreeconfig of
ImportSubTree subdir _ -> ImportTree -> fromImportLocation loc
getTopFilePath subdir P.</> fromImportLocation loc ImportSubTree subdir _ ->
getTopFilePath subdir P.</> fromImportLocation loc
getcidkey cidmap db cid = liftIO $ getcidkey cidmap db cid = liftIO $
CIDDb.getContentIdentifierKeys db rs cid >>= \case CIDDb.getContentIdentifierKeys db rs cid >>= \case

1
CHANGELOG
View file

@ -28,6 +28,7 @@ git-annex (8.20200815) UNRELEASED; urgency=medium
to 5, to avoid some unusual edge cases where too much retrying could to 5, to avoid some unusual edge cases where too much retrying could
result in far more data transfer than makes sense. result in far more data transfer than makes sense.
* Retry transfers to exporttree=yes remotes same as for other remotes. * Retry transfers to exporttree=yes remotes same as for other remotes.
* import: Retry downloads that fail, same as is done for downloads generally.
-- Joey Hess <id@joeyh.name> Fri, 14 Aug 2020 14:57:45 -0400 -- Joey Hess <id@joeyh.name> Fri, 14 Aug 2020 14:57:45 -0400

7
doc/todo/tolerate_intermittent_errors_upon___34__export__34_____40__and_probably_copy__41___to_S3/comment_3_fb3e0585f6700050db7a06e27d8e1099._comment Normal file
View file

@ -0,0 +1,7 @@
[[!comment format=mdwn
username="joey"
subject="""comment 3"""
date="2020-09-04T17:40:07Z"
content="""
Both retry now.
"""]]