From c3fa1f2b0869a87a9788ebbf881993c8093f3196 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 14 Nov 2018 11:47:40 -0400 Subject: [PATCH] avoid redundant export uploads export, sync --content: Avoid unnecessarily trying to upload files to an exporttree remote that already contains the files. When the export was origianly made in one repo and now git-annex is running in a different repo, the export database is not yet populated with information about the exportLocation of files. So, it was trying to upload the files to the export, even when it already contained them. sync --content would first download the content from the export, and then re-upload the content back. And this also led to "not available" failures for each file that was not locally present yet. Fix: Just use checkPresentExport before uploading; if it succeeds update the database. This is a surprising oversight, it's possible it fixes a reversion because I would have thought I'd have noticed this problem when originally developing exporttree remotes. This commit was sponsored by Jochen Bartl on Patreon. --- CHANGELOG | 2 ++ Command/Export.hs | 21 +++++++++++++-------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 11e2983f79..fcbadc1cf1 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -12,6 +12,8 @@ git-annex (7.20181106) UNRELEASED; urgency=medium already actually downloaded. * When an export conflict prevents accessing a special remote, be clearer about what the problem is and how to resolve it. + * export, sync --content: Avoid unnecessarily trying to upload files + to an exporttree remote that already contains the files. -- Joey Hess Tue, 06 Nov 2018 12:44:27 -0400 diff --git a/Command/Export.hs b/Command/Export.hs index 1434835ae7..6f3548587c 100644 --- a/Command/Export.hs +++ b/Command/Export.hs @@ -204,15 +204,19 @@ fillExport r ea db new = do startExport :: Remote -> ExportActions Annex -> ExportHandle -> MVar Bool -> Git.LsTree.TreeItem -> CommandStart startExport r ea db cvar ti = do ek <- exportKey (Git.LsTree.sha ti) - stopUnless (notpresent ek) $ do + stopUnless (notrecordedpresent ek) $ do showStart ("export " ++ name r) f - liftIO $ modifyMVar_ cvar (pure . const True) - next $ performExport r ea db ek af (Git.LsTree.sha ti) loc + ifM (either (const False) id <$> tryNonAsync (checkPresentExport ea (asKey ek) loc)) + ( next $ next $ cleanupExport r db ek loc False + , do + liftIO $ modifyMVar_ cvar (pure . const True) + next $ performExport r ea db ek af (Git.LsTree.sha ti) loc + ) where loc = mkExportLocation f f = getTopFilePath (Git.LsTree.file ti) af = AssociatedFile (Just f) - notpresent ek = (||) + notrecordedpresent ek = (||) <$> liftIO (notElem loc <$> getExportedLocation db (asKey ek)) -- If content was removed from the remote, the export db -- will still list it, so also check location tracking. @@ -245,13 +249,14 @@ performExport r ea db ek af contentsha loc = do liftIO $ hClose h storer tmp sha1k loc m if sent - then next $ cleanupExport r db ek loc + then next $ cleanupExport r db ek loc True else stop -cleanupExport :: Remote -> ExportHandle -> ExportKey -> ExportLocation -> CommandCleanup -cleanupExport r db ek loc = do +cleanupExport :: Remote -> ExportHandle -> ExportKey -> ExportLocation -> Bool -> CommandCleanup +cleanupExport r db ek loc sent = do liftIO $ addExportedLocation db (asKey ek) loc - logChange (asKey ek) (uuid r) InfoPresent + when sent $ + logChange (asKey ek) (uuid r) InfoPresent return True startUnexport :: Remote -> ExportActions Annex -> ExportHandle -> TopFilePath -> [Git.Sha] -> CommandStart