mirrors/git-annex
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

only delete bundles on pushEmpty

Browse source 
This avoids some apparently otherwise unsolveable problems involving
races that resulted in the manifest listing bundles that were deleted.

Removed the annex-max-git-bundles config because it can't actually
result in deleting old bundles. It would still be possible to have a
config that controls how often to do a full push, which would avoid
needing to download too many bundles on clone, as well as needing to
checkpresent too many bundles in verifyManifest. But it would need a
different name and description.
This commit is contained in:
Joey Hess 2024-05-21 10:41:48 -04:00
parent f544946b09
commit 3e7324bbcb
No known key found for this signature in database
GPG key ID: DB12DB0FF05F8F38
6 changed files with 53 additions and 103 deletions
Download patch file Download diff file Expand all files Collapse all files

52
CmdLine/GitRemoteAnnex.hs
View file

@ -272,9 +272,8 @@ fullPush' :: Manifest -> State -> Remote -> [Ref] -> Annex (Bool, State)
fullPush' oldmanifest st rmt refs =do
let bs = map Git.Bundle.fullBundleSpec refs
bundlekey <- generateAndUploadGitBundle rmt bs oldmanifest
oldmanifest' <- dropOldKeys rmt oldmanifest (/= bundlekey)
let manifest = mkManifest [bundlekey]
(inManifest oldmanifest ++ outManifest oldmanifest')
(inManifest oldmanifest ++ outManifest oldmanifest)
uploadManifest rmt manifest
return (True, st { manifestCache = Nothing })
@ -291,17 +290,11 @@ guardPush st a = catchNonAsync a $ \ex -> do
incrementalPush :: State -> Remote -> M.Map Ref Sha -> M.Map Ref Sha -> Annex (Bool, State)
incrementalPush st rmt oldtrackingrefs newtrackingrefs = guardPush st $ do
oldmanifest <- maybe (downloadManifestWhenPresent rmt) pure (manifestCache st)
if length (inManifest oldmanifest) + 1 > remoteAnnexMaxGitBundles (Remote.gitconfig rmt)
then fullPush' oldmanifest st rmt (M.keys newtrackingrefs)
else go oldmanifest
bs <- calc [] (M.toList newtrackingrefs)
bundlekey <- generateAndUploadGitBundle rmt bs oldmanifest
uploadManifest rmt (oldmanifest <> mkManifest [bundlekey] [])
return (True, st { manifestCache = Nothing })
where
go oldmanifest = do
bs <- calc [] (M.toList newtrackingrefs)
bundlekey <- generateAndUploadGitBundle rmt bs oldmanifest
oldmanifest' <- dropOldKeys rmt oldmanifest (/= bundlekey)
uploadManifest rmt (oldmanifest' <> mkManifest [bundlekey] [])
return (True, st { manifestCache = Nothing })
calc c [] = return (reverse c)
calc c ((ref, sha):refs) = case M.lookup ref oldtrackingrefs of
Just oldsha
@ -610,7 +603,7 @@ downloadManifest rmt = get mkmain >>= maybe (get mkbak) (pure . Just)
_ <- dl tmp
b <- liftIO (B.readFile tmp)
case parseManifest b of
Right m -> return (Just m)
Right m -> Just <$> verifyManifest rmt m
Left err -> giveup err
getexport _ [] = return Nothing
@ -634,13 +627,6 @@ downloadManifest rmt = get mkmain >>= maybe (get mkbak) (pure . Just)
-- So this may be interrupted and leave the manifest key not present.
-- To deal with that, there is a backup manifest key. This takes care
-- to ensure that one of the two keys will always exist.
--
-- Once the manifest has been uploaded, attempts to drop all outManifest
-- keys. A failure to drop does not cause an error to be thrown, because
-- the push has already succeeded. Avoids re-uploading the manifest with
-- the dropped keys removed from outManifest, because dropping the keys
-- takes some time and another push may have already overwritten
-- the manifest in the meantime.
uploadManifest :: Remote -> Manifest -> Annex ()
uploadManifest rmt manifest = do
ok <- ifM (Remote.checkPresent rmt mkbak)
@ -650,9 +636,8 @@ uploadManifest rmt manifest = do
-- This ensures that at no point are both deleted.
, put mkbak <&&> dropandput mkmain
)
if ok
then void $ dropOldKeys rmt manifest (const True)
else uploadfailed
unless ok
uploadfailed
where
mkmain = genManifestKey (Remote.uuid rmt)
mkbak = genBackupManifestKey (Remote.uuid rmt)
@ -696,6 +681,17 @@ dropOldKeys rmt manifest p =
mkManifest (inManifest manifest)
<$> filterM (dropKey rmt) (filter p (outManifest manifest))
-- When pushEmpty raced with another push, it could result in the manifest
-- listing bundles that it deleted. Such a manifest has to be treated the
-- same as an empty manifest. To detect that, this checks that all the
-- bundles listed in the manifest still exist on the remote.
verifyManifest :: Remote -> Manifest -> Annex Manifest
verifyManifest rmt manifest =
ifM (allM (checkPresentGitBundle rmt) (inManifest manifest))
( return manifest
, return $ mkManifest [] (inManifest manifest <> outManifest manifest)
)
-- Downloads a git bundle to the annex objects directory, unless
-- the object file is already present. Returns the filename of the object
-- file.
@ -732,6 +728,16 @@ downloadGitBundle rmt k = getKeyExportLocations rmt k >>= \case
rsp = Remote.retrievalSecurityPolicy rmt
vc = Remote.RemoteVerify rmt
-- Checks if a bundle is present. Throws errors if the remote cannot be
-- accessed.
checkPresentGitBundle :: Remote -> Key -> Annex Bool
checkPresentGitBundle rmt k =
getKeyExportLocations rmt k >>= \case
Nothing -> Remote.checkPresent rmt k
Just locs -> anyM checkexport locs
where
checkexport = Remote.checkPresentExport (Remote.exportActions rmt) k
-- Uploads a bundle or manifest object from the annex objects directory
-- to the remote.
--