Bugfix to getting content from an export remote with -J, when the export database was not yet populated.

CHANGELOG

@@ -34,6 +34,8 @@ git-annex (8.20200221) UNRELEASED; urgency=medium
   * Auto upgrades from older repo versions, like v5, now jump right to v8.
   * Extended annex.security.allowed-ip-addresses to let specific ports
     of an IP address to be used, while denying use of other ports.
+  * Bugfix to getting content from an export remote with -J, when the
+    export database was not yet populated.
 
  -- Joey Hess <id@joeyh.name>  Wed, 19 Feb 2020 12:48:58 -0400
 

Remote/Helper/ExportImport.hs

@@ -160,7 +160,6 @@ adjustExportImport r rs = case getRemoteConfigValue exportTreeField (config r) o
 				{ storeExport = \f k loc p -> do
 					db <- getciddb ciddbv
 					exportdb <- getexportdb exportdbv
-					updateexportdb exportdb exportdbv
 					oldks <- liftIO $ Export.getExportTreeKey exportdb loc
 					oldcids <- liftIO $ concat
 						<$> mapM (ContentIdentifier.getContentIdentifiers db rs) oldks
@@ -277,8 +276,7 @@ adjustExportImport r rs = case getRemoteConfigValue exportTreeField (config r) o
 		lcklckv <- liftIO newEmptyTMVarIO
 		dbv <- liftIO newEmptyTMVarIO
 		exportinconflict <- liftIO $ newTVarIO False
-		exportupdated <- liftIO $ newTMVarIO ()
-		return (dbv, lcklckv, exportinconflict, exportupdated)
+		return (dbv, lcklckv, exportinconflict)
 
 	-- Only open the database once it's needed.
 	getciddb (dbtv, lcklckv) =
@@ -300,13 +298,18 @@ adjustExportImport r rs = case getRemoteConfigValue exportTreeField (config r) o
 				)
 	
 	-- Only open the database once it's needed.
-	getexportdb (dbv, lcklckv, _, _) = 
+	--
+	-- After opening the database, check if the export log is
+	-- different than the database, and update the database, to notice
+	-- when an export has been updated from another repository.
+	getexportdb (dbv, lcklckv, exportinconflict) = 
 		liftIO (atomically (tryReadTMVar dbv)) >>= \case
 			Just db -> return db
 			-- let only one thread take the lock
 			Nothing -> ifM (liftIO $ atomically $ tryPutTMVar lcklckv ())
 				( do
 					db <- Export.openDb (uuid r)
+					updateexportdb db exportinconflict
 					liftIO $ atomically $ putTMVar dbv db
 					return db
 				-- loser waits for winner to open the db and
@@ -314,24 +317,18 @@ adjustExportImport r rs = case getRemoteConfigValue exportTreeField (config r) o
 				, liftIO $ atomically (readTMVar dbv)
 				)
 
-	getexportinconflict (_, _, v, _) = v
+	getexportinconflict (_, _, v) = v
 
-	-- Check once if the export log is different than the database and
-	-- updates the database, to notice when an export has been
-	-- updated from another repository.				
-	updateexportdb db (_, _, exportinconflict, exportupdated) =
-		liftIO (atomically (tryTakeTMVar exportupdated)) >>= \case
-			Just () -> Export.updateExportTreeFromLog db >>= \case
-				Export.ExportUpdateSuccess -> return ()
-				Export.ExportUpdateConflict -> do
-					warnExportImportConflict r
-					liftIO $ atomically $
-						writeTVar exportinconflict True
-			Nothing -> return ()
+	updateexportdb db exportinconflict =
+		Export.updateExportTreeFromLog db >>= \case
+			Export.ExportUpdateSuccess -> return ()
+			Export.ExportUpdateConflict -> do
+				warnExportImportConflict r
+				liftIO $ atomically $
+					writeTVar exportinconflict True
 		
 	getexportlocs dbv k = do
 		db <- getexportdb dbv
-		updateexportdb db dbv
 		liftIO $ Export.getExportTree db k
 
 	retrieveKeyFileFromExport dbv k _af dest p = unVerified $

doc/bugs/fails_to___96__get__96___in_parallel_for_a_freshly_clone_from_public_s3_bucket_where_versioning_info_was_forgotten.mdwn

@@ -89,3 +89,5 @@ get sub-01/meg/sub-01_task-audiovisual_run-01_meg.fif (from s3-PUBLIC...)
 
 [[!meta author=yoh]]
 [[!tag projects/repronim]]
+
+> [[fixed|done]] --[[Joey]]

doc/bugs/fails_to___96__get__96___in_parallel_for_a_freshly_clone_from_public_s3_bucket_where_versioning_info_was_forgotten/comment_4_4a4d5b62af4a42ab9072ccaa6d8bed7d._comment

@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2020-02-26T18:22:46Z"
+ content="""
+How do I produce such a repo, I thought that git-annex has fixed the 
+problem that made it not include the S3 versioning information?
+I don't want to see a lot of repos being created with that information
+missing.
+
+----
+
+Anyway, the S3 version is is a red herring, the failure is actually
+caused by the export db not getting populated from the git-annex branch
+before some threads try to use it. Remote.Helper.ExportImport has a
+updateexportdb that lets one thread update the db, but other threads
+don't block waiting for it. Easily fixed.
+"""]]