annex.skipunknown with transition plan

Added annex.skipunknown git config, that can be set to false to change the
behavior of commands like `git annex get foo*`, to not skip over files/dirs
that are not checked into git and are explicitly listed in the command
line.

Significant complexity was needed to handle git-annex add, which uses some
git ls-files calls, but needs to not use --error-unmatch because of course
the files are not known to git.

annex.skipunknown is planned to change to default to false in a
git-annex release in early 2022. There's a todo for that.

Command/Add.hs

@@ -82,10 +82,15 @@ seek o = startConcurrency commandStages $ do
 				giveup "--update --batch is not supported"
 			| otherwise -> batchFilesMatching fmt (gofile . toRawFilePath)
 		NoBatch -> do
-			l <- workTreeItems (addThese o)
-			let go a = a (commandAction . gofile) l
+			-- Avoid git ls-files complaining about files that
+			-- are not known to git yet, since this will add
+			-- them. Instead, have workTreeItems warn about other
+			-- problems, like files that don't exist.
+			let ww = WarnUnmatchWorkTreeItems
+			l <- workTreeItems ww (addThese o)
+			let go a = a ww (commandAction . gofile) l
 			unless (updateOnly o) $
-				go withFilesNotInGit
+				go (const withFilesNotInGit)
 			go withFilesMaybeModified
 			go withUnmodifiedUnlockedPointers
 

Command/Copy.hs

@@ -51,8 +51,10 @@ seek o = startConcurrency commandStages $ do
 		NoBatch -> withKeyOptions
 			(keyOptions o) (autoMode o)
 			(commandAction . Command.Move.startKey (fromToOptions o) Command.Move.RemoveNever)
-			(withFilesInGit $ commandAction . go)
-			=<< workTreeItems (copyFiles o)
+			(withFilesInGit ww $ commandAction . go)
+			=<< workTreeItems ww (copyFiles o)
+  where
+	ww = WarnUnmatchLsFiles
 
 {- A copy is just a move that does not delete the source file.
  - However, auto mode avoids unnecessary copies, and avoids getting or

Command/Drop.hs

@@ -57,10 +57,11 @@ seek o = startConcurrency commandStages $
 		Batch fmt -> batchFilesMatching fmt (go . toRawFilePath)
 		NoBatch -> withKeyOptions (keyOptions o) (autoMode o)
 			(commandAction . startKeys o)
-			(withFilesInGit (commandAction . go))
-			=<< workTreeItems (dropFiles o)
+			(withFilesInGit ww (commandAction . go))
+			=<< workTreeItems ww (dropFiles o)
   where
 	go = whenAnnexed $ start o
+	ww = WarnUnmatchLsFiles
 
 start :: DropOptions -> RawFilePath -> Key -> CommandStart
 start o file key = start' o key afile ai

Command/Find.hs

@@ -57,11 +57,12 @@ seek :: FindOptions -> CommandSeek
 seek o = case batchOption o of
 	NoBatch -> withKeyOptions (keyOptions o) False
 		(commandAction . startKeys o)
-		(withFilesInGit (commandAction . go))
-		=<< workTreeItems (findThese o)
+		(withFilesInGit ww (commandAction . go))
+		=<< workTreeItems ww (findThese o)
 	Batch fmt -> batchFilesMatching fmt (go . toRawFilePath)
   where
 	go = whenAnnexed $ start o
+	ww = WarnUnmatchLsFiles
 
 -- only files inAnnex are shown, unless the user has requested
 -- others via a limit

Command/Fix.hs

@@ -32,9 +32,11 @@ cmd = noCommit $ withGlobalOptions [annexedMatchingOptions] $
 
 seek :: CmdParams -> CommandSeek
 seek ps = unlessM crippledFileSystem $ do 
-	withFilesInGit
+	withFilesInGit ww
 		(commandAction . (whenAnnexed $ start FixAll))
-		=<< workTreeItems ps
+		=<< workTreeItems ww ps
+  where
+	ww = WarnUnmatchLsFiles
 
 data FixWhat = FixSymlinks | FixAll
 

Command/Fsck.hs

@@ -94,10 +94,12 @@ seek o = startConcurrency commandStages $ do
 	i <- prepIncremental u (incrementalOpt o)
 	withKeyOptions (keyOptions o) False
 		(\kai -> commandAction . startKey from i kai =<< getNumCopies)
-		(withFilesInGit $ commandAction . (whenAnnexed (start from i)))
-		=<< workTreeItems (fsckFiles o)
+		(withFilesInGit ww $ commandAction . (whenAnnexed (start from i)))
+		=<< workTreeItems ww (fsckFiles o)
 	cleanupIncremental i
 	void $ tryIO $ recordActivity Fsck u
+  where
+	ww = WarnUnmatchLsFiles
 
 checkDeadRepo :: UUID -> Annex ()
 checkDeadRepo u =

Command/Get.hs

@@ -45,8 +45,10 @@ seek o = startConcurrency downloadStages $ do
 		Batch fmt -> batchFilesMatching fmt (go . toRawFilePath)
 		NoBatch -> withKeyOptions (keyOptions o) (autoMode o)
 			(commandAction . startKeys from)
-			(withFilesInGit (commandAction . go))
-			=<< workTreeItems (getFiles o)
+			(withFilesInGit ww (commandAction . go))
+			=<< workTreeItems ww (getFiles o)
+  where
+	ww = WarnUnmatchLsFiles
 
 start :: GetOptions -> Maybe Remote -> RawFilePath -> Key -> CommandStart
 start o from file key = start' expensivecheck from key afile ai

Command/Inprogress.hs

@@ -38,9 +38,11 @@ seek o = do
 			| otherwise -> commandAction stop
 		_ -> do
 			let s = S.fromList ts
-			withFilesInGit
+			withFilesInGit ww
 				(commandAction . (whenAnnexed (start s)))
-				=<< workTreeItems (inprogressFiles o)
+				=<< workTreeItems ww (inprogressFiles o)
+  where
+	ww = WarnUnmatchLsFiles
 
 start :: S.Set Key -> RawFilePath -> Key -> CommandStart
 start s _file k

Command/List.hs

@@ -44,9 +44,10 @@ seek :: ListOptions -> CommandSeek
 seek o = do
 	list <- getList o
 	printHeader list
-	withFilesInGit
-		(commandAction . (whenAnnexed $ start list))
-		=<< workTreeItems (listThese o)
+	withFilesInGit ww (commandAction . (whenAnnexed $ start list))
+		=<< workTreeItems ww (listThese o)
+  where
+	ww = WarnUnmatchLsFiles
 
 getList :: ListOptions -> Annex [(UUID, RemoteName, TrustLevel)]
 getList o

Command/Lock.hs

@@ -30,8 +30,10 @@ cmd = withGlobalOptions [jsonOptions, annexedMatchingOptions] $
 
 seek :: CmdParams -> CommandSeek
 seek ps = do
-	l <- workTreeItems ps
-	withFilesInGit (commandAction . (whenAnnexed startNew)) l
+	l <- workTreeItems ww ps
+	withFilesInGit ww (commandAction . (whenAnnexed startNew)) l
+  where
+	ww = WarnUnmatchLsFiles
 
 startNew :: RawFilePath -> Key -> CommandStart
 startNew file key = ifM (isJust <$> isAnnexLink file)

Command/Log.hs

@@ -86,11 +86,13 @@ seek o = do
 	zone <- liftIO getCurrentTimeZone
 	let outputter = mkOutputter m zone o
 	case (logFiles o, allOption o) of
-		(fs, False) -> withFilesInGit
+		(fs, False) -> withFilesInGit ww
 			(commandAction . (whenAnnexed $ start o outputter)) 
-			=<< workTreeItems fs
+			=<< workTreeItems ww fs
 		([], True) -> commandAction (startAll o outputter)
 		(_, True) -> giveup "Cannot specify both files and --all"
+  where
+	ww = WarnUnmatchLsFiles
 
 start :: LogOptions -> (FilePath -> Outputter) -> RawFilePath -> Key -> CommandStart
 start o outputter file key = do

Command/LookupKey.hs

@@ -31,7 +31,7 @@ run _ file = seekSingleGitFile file >>= \case
 -- But, this plumbing command does not recurse through directories.
 seekSingleGitFile :: FilePath -> Annex (Maybe RawFilePath)
 seekSingleGitFile file = do
-	(l, cleanup) <- inRepo (Git.LsFiles.inRepo [toRawFilePath file])
+	(l, cleanup) <- inRepo (Git.LsFiles.inRepo [] [toRawFilePath file])
 	r <- case l of
 		(f:[]) | takeFileName (fromRawFilePath f) == takeFileName file ->
 			return (Just f)

Command/MetaData.hs

@@ -75,15 +75,16 @@ seek :: MetaDataOptions -> CommandSeek
 seek o = case batchOption o of
 	NoBatch -> do
 		c <- liftIO currentVectorClock
+		let ww = WarnUnmatchLsFiles
 		let seeker = case getSet o of
-			Get _ -> withFilesInGit
-			GetAll -> withFilesInGit
-			Set _ -> withFilesInGitNonRecursive
+			Get _ -> withFilesInGit ww
+			GetAll -> withFilesInGit ww
+			Set _ -> withFilesInGitNonRecursive ww
 				"Not recursively setting metadata. Use --force to do that."
 		withKeyOptions (keyOptions o) False
 			(commandAction . startKeys c o)
 			(seeker (commandAction . (whenAnnexed (start c o))))
-			=<< workTreeItems (forFiles o)
+			=<< workTreeItems ww (forFiles o)
 	Batch fmt -> withMessageState $ \s -> case outputType s of
 		JSONOutput _ -> ifM limited
 			( giveup "combining --batch with file matching options is not currently supported"

Command/Migrate.hs

@@ -26,7 +26,10 @@ cmd = withGlobalOptions [annexedMatchingOptions] $
 		paramPaths (withParams seek)
 
 seek :: CmdParams -> CommandSeek
-seek = withFilesInGit (commandAction . (whenAnnexed start)) <=< workTreeItems
+seek = withFilesInGit ww (commandAction . (whenAnnexed start))
+	<=< workTreeItems ww
+  where
+	ww = WarnUnmatchLsFiles
 
 start :: RawFilePath -> Key -> CommandStart
 start file key = do

Command/Mirror.hs

@@ -44,12 +44,13 @@ seek :: MirrorOptions -> CommandSeek
 seek o = startConcurrency stages $ 
 	withKeyOptions (keyOptions o) False
 		(commandAction . startKey o (AssociatedFile Nothing))
-		(withFilesInGit (commandAction . (whenAnnexed $ start o)))
-		=<< workTreeItems (mirrorFiles o)
+		(withFilesInGit ww (commandAction . (whenAnnexed $ start o)))
+		=<< workTreeItems ww (mirrorFiles o)
   where
 	stages = case fromToOptions o of
 		FromRemote _ -> downloadStages
 		ToRemote _ -> commandStages
+	ww = WarnUnmatchLsFiles
 
 start :: MirrorOptions -> RawFilePath -> Key -> CommandStart
 start o file k = startKey o afile (k, ai)

Command/Move.hs

@@ -60,13 +60,14 @@ seek o = startConcurrency stages $ do
 		Batch fmt -> batchFilesMatching fmt (go . toRawFilePath)
 		NoBatch -> withKeyOptions (keyOptions o) False
 			(commandAction . startKey (fromToOptions o) (removeWhen o))
-			(withFilesInGit (commandAction . go))
-			=<< workTreeItems (moveFiles o)
+			(withFilesInGit ww (commandAction . go))
+			=<< workTreeItems ww (moveFiles o)
   where
 	stages = case fromToOptions o of
 		Right (FromRemote _) -> downloadStages
 		Right (ToRemote _) -> commandStages
 		Left ToHere -> downloadStages
+	ww = WarnUnmatchLsFiles
 
 start :: FromToHereOptions -> RemoveWhen -> RawFilePath -> Key -> CommandStart
 start fromto removewhen f k = start' fromto removewhen afile k ai

Command/Multicast.hs

@@ -129,7 +129,9 @@ send ups fs = do
 	-- expensive.
 	starting "sending files" (ActionItemOther Nothing) $
 		withTmpFile "send" $ \t h -> do
-			fs' <- seekHelper LsFiles.inRepo =<< workTreeItems fs
+			let ww = WarnUnmatchLsFiles
+			fs' <- seekHelper ww LsFiles.inRepo
+				=<< workTreeItems ww fs
 			matcher <- Limit.getMatcher
 			let addlist f o = whenM (matcher $ MatchingFile $ FileInfo f f) $
 				liftIO $ hPutStrLn h o

Command/PreCommit.hs

@@ -32,7 +32,8 @@ cmd = command "pre-commit" SectionPlumbing
 
 seek :: CmdParams -> CommandSeek
 seek ps = do
-	l <- workTreeItems ps
+	let ww = WarnUnmatchWorkTreeItems
+	l <- workTreeItems ww ps
 	-- fix symlinks to files being committed
 	flip withFilesToBeCommitted l $ \f -> commandAction $
 		maybe stop (Command.Fix.start Command.Fix.FixSymlinks f)

Command/Sync.hs

@@ -636,11 +636,11 @@ seekSyncContent o rs currbranch = do
 		Just WantAllKeys -> Just <$> genBloomFilter (seekworktree mvar [])
 		_ -> case currbranch of
                 	(Just origbranch, Just adj) | adjustmentHidesFiles adj -> do
-				l <- workTreeItems' (AllowHidden True) (contentOfOption o)
+				l <- workTreeItems' (AllowHidden True) ww (contentOfOption o)
 				seekincludinghidden origbranch mvar l (const noop)
 				pure Nothing
 			_ -> do
-				l <- workTreeItems (contentOfOption o)
+				l <- workTreeItems ww (contentOfOption o)
 				seekworktree mvar l (const noop)
 				pure Nothing
 	withKeyOptions' (keyOptions o) False
@@ -651,13 +651,15 @@ seekSyncContent o rs currbranch = do
 	liftIO $ not <$> isEmptyMVar mvar
   where
 	seekworktree mvar l bloomfeeder = 
-		seekHelper LsFiles.inRepo l
+		seekHelper ww LsFiles.inRepo l
 			>>= gofiles bloomfeeder mvar
 
 	seekincludinghidden origbranch mvar l bloomfeeder = 
-		seekHelper (LsFiles.inRepoOrBranch origbranch) l 
+		seekHelper ww (LsFiles.inRepoOrBranch origbranch) l 
 			>>= gofiles bloomfeeder mvar
 
+	ww = WarnUnmatchLsFiles
+
 	gofiles bloomfeeder mvar = mapM_ $ \f ->
 		ifAnnexed f
 			(go (Right bloomfeeder) mvar (AssociatedFile (Just f)))

Command/Unannex.hs

@@ -23,7 +23,10 @@ cmd = withGlobalOptions [annexedMatchingOptions] $
 		paramPaths (withParams seek)
 
 seek :: CmdParams -> CommandSeek
-seek ps = (withFilesInGit $ commandAction . whenAnnexed start) =<< workTreeItems ps
+seek ps = (withFilesInGit ww $ commandAction . whenAnnexed start)
+	=<< workTreeItems ww ps
+  where
+	ww = WarnUnmatchLsFiles
 
 start :: RawFilePath -> Key -> CommandStart
 start file key = stopUnless (inAnnex key) $

Command/Undo.hs

@@ -27,7 +27,7 @@ seek :: CmdParams -> CommandSeek
 seek ps = do
 	-- Safety first; avoid any undo that would touch files that are not
 	-- in the index.
-	(fs, cleanup) <- inRepo $ LsFiles.notInRepo False (map toRawFilePath ps)
+	(fs, cleanup) <- inRepo $ LsFiles.notInRepo [] False (map toRawFilePath ps)
 	unless (null fs) $
 		giveup $ "Cannot undo changes to files that are not checked into git: " ++ unwords (map fromRawFilePath fs)
 	void $ liftIO $ cleanup

Command/Uninit.hs

@@ -41,11 +41,13 @@ check = do
 
 seek :: CmdParams -> CommandSeek
 seek ps = do
-	l <- workTreeItems ps
+	l <- workTreeItems ww ps
 	withFilesNotInGit (commandAction . whenAnnexed (startCheckIncomplete . fromRawFilePath)) l
 	Annex.changeState $ \s -> s { Annex.fast = True }
-	withFilesInGit (commandAction . whenAnnexed Command.Unannex.start) l
+	withFilesInGit ww (commandAction . whenAnnexed Command.Unannex.start) l
 	finish
+  where
+	ww = WarnUnmatchLsFiles
 
 {- git annex symlinks that are not checked into git could be left by an
  - interrupted add. -}

Command/Unlock.hs

@@ -27,7 +27,10 @@ mkcmd n d = withGlobalOptions [jsonOptions, annexedMatchingOptions] $
 	command n SectionCommon d paramPaths (withParams seek)
 
 seek :: CmdParams -> CommandSeek
-seek ps = withFilesInGit (commandAction . whenAnnexed start) =<< workTreeItems ps
+seek ps = withFilesInGit ww (commandAction . whenAnnexed start)
+	=<< workTreeItems ww ps
+  where
+	ww = WarnUnmatchLsFiles
 
 start :: RawFilePath -> Key -> CommandStart
 start file key = ifM (isJust <$> isAnnexLink file)

Command/Unused.hs

@@ -210,9 +210,9 @@ withKeysReferenced' mdir initial a = do
 			( return ([], return True)
 			, do
 				top <- fromRepo Git.repoPath
-				inRepo $ LsFiles.allFiles [top]
+				inRepo $ LsFiles.allFiles [] [top]
 			)
-		Just dir -> inRepo $ LsFiles.inRepo [toRawFilePath dir]
+		Just dir -> inRepo $ LsFiles.inRepo [] [toRawFilePath dir]
 	go v [] = return v
 	go v (f:fs) = do
 		mk <- lookupFile f

Command/View.hs

@@ -101,7 +101,7 @@ checkoutViewBranch view mkbranch = do
 		 - removed.) -}
 		top <- liftIO . absPath . fromRawFilePath =<< fromRepo Git.repoPath
 		(l, cleanup) <- inRepo $
-			LsFiles.notInRepoIncludingEmptyDirectories False
+			LsFiles.notInRepoIncludingEmptyDirectories [] False
 				[toRawFilePath top]
 		forM_ l (removeemptydir top)
 		liftIO $ void cleanup

Command/Whereis.hs

@@ -57,8 +57,10 @@ seek o = do
 		NoBatch -> 
 			withKeyOptions (keyOptions o) False
 				(commandAction . startKeys o m)
-				(withFilesInGit (commandAction . go))
-				=<< workTreeItems (whereisFiles o)
+				(withFilesInGit ww (commandAction . go))
+				=<< workTreeItems ww (whereisFiles o)
+  where
+	ww = WarnUnmatchLsFiles
 
 start :: WhereisOptions -> M.Map UUID Remote -> RawFilePath -> Key -> CommandStart
 start o remotemap file key =