scanAnnexedFiles in smudge --update

This makes git checkout and git merge hooks do the work to catch up with
changes that they made to the tree. Rather than doing it at some later
point when the user is not thinking about that past operation.

Sponsored-by: Dartmouth College's Datalad project

Annex/Init.hs

@@ -37,7 +37,6 @@ import Annex.UUID
 import Annex.WorkTree
 import Annex.Fixup
 import Annex.Path
-import Annex.Concurrent
 import Config
 import Config.Files
 import Config.Smudge
@@ -134,8 +133,7 @@ initialize' mversion = checkInitializeAllowed $ do
 		then configureSmudgeFilter
 		else deconfigureSmudgeFilter
 	unlessM isBareRepo $ do
-		showSideActionAfter oneSecond "scanning for annexed files" $
+		scanAnnexedFiles True
-			scanAnnexedFiles
 		hookWrite postCheckoutHook
 		hookWrite postMergeHook
 	AdjustedBranch.checkAdjustedClone >>= \case

Annex/WorkTree.hs

@@ -15,6 +15,8 @@ import Annex.Content
 import Annex.ReplaceFile
 import Annex.CurrentBranch
 import Annex.InodeSentinal
+import Annex.Concurrent
+import Utility.ThreadScheduler
 import Utility.InodeCache
 import Git.FilePath
 import Git.CatFile
@@ -78,8 +80,8 @@ ifAnnexed file yes no = maybe no yes =<< lookupKey file
  - But if worktree file does not have a pointer file's content, it is left
  - as-is.
  -}
-scanAnnexedFiles :: Annex ()
+scanAnnexedFiles :: Bool -> Annex ()
-scanAnnexedFiles = whenM (inRepo Git.Ref.headExists <&&> not <$> isBareRepo) $ do
+scanAnnexedFiles initscan = showSideActionAfter oneSecond "scanning for annexed files" $ do
 	-- This gets the keys database populated with all annexed files,
 	-- by running Database.Keys.reconcileStaged.
 	Database.Keys.runWriter (const noop)
@@ -88,14 +90,19 @@ scanAnnexedFiles = whenM (inRepo Git.Ref.headExists <&&> not <$> isBareRepo) $ d
 	-- annex object file already exists, but its inode is not yet
 	-- cached and annex.thin is set. So, the rest of this makes
 	-- another pass over the tree to do that.
-	whenM (annexThin <$> Annex.getGitConfig) $ do
+	whenM
-		g <- Annex.gitRepo
+		( pure initscan
-		(l, cleanup) <- inRepo $ Git.LsTree.lsTree
+		<&&> annexThin <$> Annex.getGitConfig
-			Git.LsTree.LsTreeRecursive
+		<&&> inRepo Git.Ref.headExists
-			(Git.LsTree.LsTreeLong True)
+		<&&> not <$> isBareRepo
-			Git.Ref.headRef
+		) $ do
-		catObjectStreamLsTree l want g go
+			g <- Annex.gitRepo
-		liftIO $ void cleanup
+			(l, cleanup) <- inRepo $ Git.LsTree.lsTree
+				Git.LsTree.LsTreeRecursive
+				(Git.LsTree.LsTreeLong True)
+				Git.Ref.headRef
+			catObjectStreamLsTree l want g go
+			liftIO $ void cleanup
   where
 	-- Want to process symlinks, and regular files.
 	want i = case Git.Types.toTreeItemType (Git.LsTree.mode i) of

Command/Smudge.hs

@@ -13,6 +13,7 @@ import Annex.Link
 import Annex.FileMatcher
 import Annex.Ingest
 import Annex.CatFile
+import Annex.WorkTree
 import Logs.Smudge
 import Logs.Location
 import qualified Database.Keys
@@ -262,6 +263,11 @@ getMoveRaceRecovery k file = void $ tryNonAsync $
 
 update :: CommandStart
 update = do
+	-- This gets run after a git checkout or merge, so it's a good
+	-- point to refresh the keys database for changes to annexed files.
+	-- Doing it explicitly here avoids a later pause in the middle of
+	-- some other action.
+	scanAnnexedFiles False
 	updateSmudged (Restage True)
 	stop
 

Upgrade/V5.hs

@@ -47,7 +47,7 @@ upgrade automatic = flip catchNonAsync onexception $ do
 		, do
 			checkGitVersionForIndirectUpgrade
 		)
-	scanAnnexedFiles
+	scanAnnexedFiles True
 	configureSmudgeFilter
 	-- Inode sentinal file was only used in direct mode and when
 	-- locking down files as they were added. In v6, it's used more

doc/todo/display_when_reconcileStaged_is_taking_a_long_time/comment_1_7efa1d29b475b445cea6fe44d402b275._comment

@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2021-06-08T15:21:02Z"
+ content="""
+Made `git-annex smudge --update` run the scan, and so the post-checkout or
+post-merge hook will call it. 
+
+That avoids the scenario shown above. But adding a lot of files to the
+index can still cause a later pause for reconcileStaged without indication
+what it's doing.
+"""]]