fix parallel copy from/to a local git repo

Improve handling of parallelization with -J when copying content from/to a
git remote that is a local path.

Sponsored-by: Nicholas Golder-Manning on Patreon

CHANGELOG

@@ -1,5 +1,7 @@
 git-annex (10.20220625) UNRELEASED; urgency=medium
 
+  * Improve handling of parallelization with -J when copying content
+    from/to a git remote that is a local path.
   * stack.yaml: Updated to lts-19.13
 
  -- Joey Hess <id@joeyh.name>  Tue, 28 Jun 2022 14:49:17 -0400

Remote/Git.hs

@@ -609,12 +609,12 @@ repairRemote r a = return $ do
 		ensureInitialized
 		a `finally` stopCoProcesses
 
-data LocalRemoteAnnex = LocalRemoteAnnex Git.Repo (MVar (Maybe (Annex.AnnexState, Annex.AnnexRead)))
+data LocalRemoteAnnex = LocalRemoteAnnex Git.Repo (MVar [(Annex.AnnexState, Annex.AnnexRead)])
 
 {- This can safely be called on a Repo that is not local, but of course
  - onLocal will not work if used with the result. -}
 mkLocalRemoteAnnex :: Git.Repo -> Annex (LocalRemoteAnnex)
-mkLocalRemoteAnnex repo = LocalRemoteAnnex repo <$> liftIO (newMVar Nothing)
+mkLocalRemoteAnnex repo = LocalRemoteAnnex repo <$> liftIO (newMVar [])
 
 {- Runs an action from the perspective of a local remote.
  -
@@ -645,10 +645,13 @@ newLocal repo = do
 
 onLocal' :: LocalRemoteAnnex -> Annex a -> Annex a
 onLocal' (LocalRemoteAnnex repo mv) a = liftIO (takeMVar mv) >>= \case
-	Nothing -> do
+	[] -> do
+		liftIO $ putMVar mv []
 		v <- newLocal repo
 		go (v, ensureInitialized >> a)
-	Just v -> go (v, a)
+	(v:rest) -> do
+		liftIO $ putMVar mv rest
+		go (v, a)
   where
 	go ((st, rd), a') = do
 		curro <- Annex.getState Annex.output
@@ -657,7 +660,9 @@ onLocal' (LocalRemoteAnnex repo mv) a = liftIO (takeMVar mv) >>= \case
 		(ret, (st', _rd)) <- liftIO $ act `onException` cache (st, rd)
 		liftIO $ cache (st', rd)
 		return ret
-	cache = putMVar mv . Just
+	cache v = do
+		l <- takeMVar mv
+		putMVar mv (v:l)
 
 {- Faster variant of onLocal.
  -

doc/bugs/copy_-J_--to_local_git_remote_does_not_run_concurrently.mdwn

@@ -24,3 +24,13 @@ would make sense to.)
 
 git-annex version: 8.20201103 --[[Joey]]
 
+> Reproduced with 10.20220625. 3 100 mb files and copy to a local git repo.
+> Also, `copy --from` and `get` do have the problem, at least they do now.
+> --[[Joey]]
+
+> The problem is that Remote.Git.onLocal uses a MVar for the remote state,
+> and the copy runs in that. So while one copy is running, the rest block.
+> So fixing this will need a pool of local remote states.
+> --[[Joey]]
+
+>> [[fixed|done]] --[[Joey]]