Bugfix to getting content from an export remote with -J, when the export database was not yet populated.

This commit is contained in:
Joey Hess 2020-02-26 14:57:29 -04:00
parent cb86caf929
commit e520341500
No known key found for this signature in database
GPG key ID: DB12DB0FF05F8F38
4 changed files with 37 additions and 18 deletions

View file

@ -34,6 +34,8 @@ git-annex (8.20200221) UNRELEASED; urgency=medium
* Auto upgrades from older repo versions, like v5, now jump right to v8. * Auto upgrades from older repo versions, like v5, now jump right to v8.
* Extended annex.security.allowed-ip-addresses to let specific ports * Extended annex.security.allowed-ip-addresses to let specific ports
of an IP address to be used, while denying use of other ports. of an IP address to be used, while denying use of other ports.
* Bugfix to getting content from an export remote with -J, when the
export database was not yet populated.
-- Joey Hess <id@joeyh.name> Wed, 19 Feb 2020 12:48:58 -0400 -- Joey Hess <id@joeyh.name> Wed, 19 Feb 2020 12:48:58 -0400

View file

@ -160,7 +160,6 @@ adjustExportImport r rs = case getRemoteConfigValue exportTreeField (config r) o
{ storeExport = \f k loc p -> do { storeExport = \f k loc p -> do
db <- getciddb ciddbv db <- getciddb ciddbv
exportdb <- getexportdb exportdbv exportdb <- getexportdb exportdbv
updateexportdb exportdb exportdbv
oldks <- liftIO $ Export.getExportTreeKey exportdb loc oldks <- liftIO $ Export.getExportTreeKey exportdb loc
oldcids <- liftIO $ concat oldcids <- liftIO $ concat
<$> mapM (ContentIdentifier.getContentIdentifiers db rs) oldks <$> mapM (ContentIdentifier.getContentIdentifiers db rs) oldks
@ -277,8 +276,7 @@ adjustExportImport r rs = case getRemoteConfigValue exportTreeField (config r) o
lcklckv <- liftIO newEmptyTMVarIO lcklckv <- liftIO newEmptyTMVarIO
dbv <- liftIO newEmptyTMVarIO dbv <- liftIO newEmptyTMVarIO
exportinconflict <- liftIO $ newTVarIO False exportinconflict <- liftIO $ newTVarIO False
exportupdated <- liftIO $ newTMVarIO () return (dbv, lcklckv, exportinconflict)
return (dbv, lcklckv, exportinconflict, exportupdated)
-- Only open the database once it's needed. -- Only open the database once it's needed.
getciddb (dbtv, lcklckv) = getciddb (dbtv, lcklckv) =
@ -300,13 +298,18 @@ adjustExportImport r rs = case getRemoteConfigValue exportTreeField (config r) o
) )
-- Only open the database once it's needed. -- Only open the database once it's needed.
getexportdb (dbv, lcklckv, _, _) = --
-- After opening the database, check if the export log is
-- different than the database, and update the database, to notice
-- when an export has been updated from another repository.
getexportdb (dbv, lcklckv, exportinconflict) =
liftIO (atomically (tryReadTMVar dbv)) >>= \case liftIO (atomically (tryReadTMVar dbv)) >>= \case
Just db -> return db Just db -> return db
-- let only one thread take the lock -- let only one thread take the lock
Nothing -> ifM (liftIO $ atomically $ tryPutTMVar lcklckv ()) Nothing -> ifM (liftIO $ atomically $ tryPutTMVar lcklckv ())
( do ( do
db <- Export.openDb (uuid r) db <- Export.openDb (uuid r)
updateexportdb db exportinconflict
liftIO $ atomically $ putTMVar dbv db liftIO $ atomically $ putTMVar dbv db
return db return db
-- loser waits for winner to open the db and -- loser waits for winner to open the db and
@ -314,24 +317,18 @@ adjustExportImport r rs = case getRemoteConfigValue exportTreeField (config r) o
, liftIO $ atomically (readTMVar dbv) , liftIO $ atomically (readTMVar dbv)
) )
getexportinconflict (_, _, v, _) = v getexportinconflict (_, _, v) = v
-- Check once if the export log is different than the database and updateexportdb db exportinconflict =
-- updates the database, to notice when an export has been Export.updateExportTreeFromLog db >>= \case
-- updated from another repository. Export.ExportUpdateSuccess -> return ()
updateexportdb db (_, _, exportinconflict, exportupdated) = Export.ExportUpdateConflict -> do
liftIO (atomically (tryTakeTMVar exportupdated)) >>= \case warnExportImportConflict r
Just () -> Export.updateExportTreeFromLog db >>= \case liftIO $ atomically $
Export.ExportUpdateSuccess -> return () writeTVar exportinconflict True
Export.ExportUpdateConflict -> do
warnExportImportConflict r
liftIO $ atomically $
writeTVar exportinconflict True
Nothing -> return ()
getexportlocs dbv k = do getexportlocs dbv k = do
db <- getexportdb dbv db <- getexportdb dbv
updateexportdb db dbv
liftIO $ Export.getExportTree db k liftIO $ Export.getExportTree db k
retrieveKeyFileFromExport dbv k _af dest p = unVerified $ retrieveKeyFileFromExport dbv k _af dest p = unVerified $

View file

@ -89,3 +89,5 @@ get sub-01/meg/sub-01_task-audiovisual_run-01_meg.fif (from s3-PUBLIC...)
[[!meta author=yoh]] [[!meta author=yoh]]
[[!tag projects/repronim]] [[!tag projects/repronim]]
> [[fixed|done]] --[[Joey]]

View file

@ -0,0 +1,18 @@
[[!comment format=mdwn
username="joey"
subject="""comment 4"""
date="2020-02-26T18:22:46Z"
content="""
How do I produce such a repo, I thought that git-annex has fixed the
problem that made it not include the S3 versioning information?
I don't want to see a lot of repos being created with that information
missing.
----
Anyway, the S3 version is is a red herring, the failure is actually
caused by the export db not getting populated from the git-annex branch
before some threads try to use it. Remote.Helper.ExportImport has a
updateexportdb that lets one thread update the db, but other threads
don't block waiting for it. Easily fixed.
"""]]