annex.skipunknown with transition plan

Added annex.skipunknown git config, that can be set to false to change the
behavior of commands like `git annex get foo*`, to not skip over files/dirs
that are not checked into git and are explicitly listed in the command
line.

Significant complexity was needed to handle git-annex add, which uses some
git ls-files calls, but needs to not use --error-unmatch because of course
the files are not known to git.

annex.skipunknown is planned to change to default to false in a
git-annex release in early 2022. There's a todo for that.
This commit is contained in:
Joey Hess 2020-05-28 15:55:17 -04:00
parent 5b28a37ea1
commit 89b2542d3c
No known key found for this signature in database
GPG key ID: DB12DB0FF05F8F38
42 changed files with 271 additions and 169 deletions

View file

@ -82,10 +82,15 @@ seek o = startConcurrency commandStages $ do
giveup "--update --batch is not supported"
| otherwise -> batchFilesMatching fmt (gofile . toRawFilePath)
NoBatch -> do
l <- workTreeItems (addThese o)
let go a = a (commandAction . gofile) l
-- Avoid git ls-files complaining about files that
-- are not known to git yet, since this will add
-- them. Instead, have workTreeItems warn about other
-- problems, like files that don't exist.
let ww = WarnUnmatchWorkTreeItems
l <- workTreeItems ww (addThese o)
let go a = a ww (commandAction . gofile) l
unless (updateOnly o) $
go withFilesNotInGit
go (const withFilesNotInGit)
go withFilesMaybeModified
go withUnmodifiedUnlockedPointers

View file

@ -51,8 +51,10 @@ seek o = startConcurrency commandStages $ do
NoBatch -> withKeyOptions
(keyOptions o) (autoMode o)
(commandAction . Command.Move.startKey (fromToOptions o) Command.Move.RemoveNever)
(withFilesInGit $ commandAction . go)
=<< workTreeItems (copyFiles o)
(withFilesInGit ww $ commandAction . go)
=<< workTreeItems ww (copyFiles o)
where
ww = WarnUnmatchLsFiles
{- A copy is just a move that does not delete the source file.
- However, auto mode avoids unnecessary copies, and avoids getting or

View file

@ -57,10 +57,11 @@ seek o = startConcurrency commandStages $
Batch fmt -> batchFilesMatching fmt (go . toRawFilePath)
NoBatch -> withKeyOptions (keyOptions o) (autoMode o)
(commandAction . startKeys o)
(withFilesInGit (commandAction . go))
=<< workTreeItems (dropFiles o)
(withFilesInGit ww (commandAction . go))
=<< workTreeItems ww (dropFiles o)
where
go = whenAnnexed $ start o
ww = WarnUnmatchLsFiles
start :: DropOptions -> RawFilePath -> Key -> CommandStart
start o file key = start' o key afile ai

View file

@ -57,11 +57,12 @@ seek :: FindOptions -> CommandSeek
seek o = case batchOption o of
NoBatch -> withKeyOptions (keyOptions o) False
(commandAction . startKeys o)
(withFilesInGit (commandAction . go))
=<< workTreeItems (findThese o)
(withFilesInGit ww (commandAction . go))
=<< workTreeItems ww (findThese o)
Batch fmt -> batchFilesMatching fmt (go . toRawFilePath)
where
go = whenAnnexed $ start o
ww = WarnUnmatchLsFiles
-- only files inAnnex are shown, unless the user has requested
-- others via a limit

View file

@ -32,9 +32,11 @@ cmd = noCommit $ withGlobalOptions [annexedMatchingOptions] $
seek :: CmdParams -> CommandSeek
seek ps = unlessM crippledFileSystem $ do
withFilesInGit
withFilesInGit ww
(commandAction . (whenAnnexed $ start FixAll))
=<< workTreeItems ps
=<< workTreeItems ww ps
where
ww = WarnUnmatchLsFiles
data FixWhat = FixSymlinks | FixAll

View file

@ -94,10 +94,12 @@ seek o = startConcurrency commandStages $ do
i <- prepIncremental u (incrementalOpt o)
withKeyOptions (keyOptions o) False
(\kai -> commandAction . startKey from i kai =<< getNumCopies)
(withFilesInGit $ commandAction . (whenAnnexed (start from i)))
=<< workTreeItems (fsckFiles o)
(withFilesInGit ww $ commandAction . (whenAnnexed (start from i)))
=<< workTreeItems ww (fsckFiles o)
cleanupIncremental i
void $ tryIO $ recordActivity Fsck u
where
ww = WarnUnmatchLsFiles
checkDeadRepo :: UUID -> Annex ()
checkDeadRepo u =

View file

@ -45,8 +45,10 @@ seek o = startConcurrency downloadStages $ do
Batch fmt -> batchFilesMatching fmt (go . toRawFilePath)
NoBatch -> withKeyOptions (keyOptions o) (autoMode o)
(commandAction . startKeys from)
(withFilesInGit (commandAction . go))
=<< workTreeItems (getFiles o)
(withFilesInGit ww (commandAction . go))
=<< workTreeItems ww (getFiles o)
where
ww = WarnUnmatchLsFiles
start :: GetOptions -> Maybe Remote -> RawFilePath -> Key -> CommandStart
start o from file key = start' expensivecheck from key afile ai

View file

@ -38,9 +38,11 @@ seek o = do
| otherwise -> commandAction stop
_ -> do
let s = S.fromList ts
withFilesInGit
withFilesInGit ww
(commandAction . (whenAnnexed (start s)))
=<< workTreeItems (inprogressFiles o)
=<< workTreeItems ww (inprogressFiles o)
where
ww = WarnUnmatchLsFiles
start :: S.Set Key -> RawFilePath -> Key -> CommandStart
start s _file k

View file

@ -44,9 +44,10 @@ seek :: ListOptions -> CommandSeek
seek o = do
list <- getList o
printHeader list
withFilesInGit
(commandAction . (whenAnnexed $ start list))
=<< workTreeItems (listThese o)
withFilesInGit ww (commandAction . (whenAnnexed $ start list))
=<< workTreeItems ww (listThese o)
where
ww = WarnUnmatchLsFiles
getList :: ListOptions -> Annex [(UUID, RemoteName, TrustLevel)]
getList o

View file

@ -30,8 +30,10 @@ cmd = withGlobalOptions [jsonOptions, annexedMatchingOptions] $
seek :: CmdParams -> CommandSeek
seek ps = do
l <- workTreeItems ps
withFilesInGit (commandAction . (whenAnnexed startNew)) l
l <- workTreeItems ww ps
withFilesInGit ww (commandAction . (whenAnnexed startNew)) l
where
ww = WarnUnmatchLsFiles
startNew :: RawFilePath -> Key -> CommandStart
startNew file key = ifM (isJust <$> isAnnexLink file)

View file

@ -86,11 +86,13 @@ seek o = do
zone <- liftIO getCurrentTimeZone
let outputter = mkOutputter m zone o
case (logFiles o, allOption o) of
(fs, False) -> withFilesInGit
(fs, False) -> withFilesInGit ww
(commandAction . (whenAnnexed $ start o outputter))
=<< workTreeItems fs
=<< workTreeItems ww fs
([], True) -> commandAction (startAll o outputter)
(_, True) -> giveup "Cannot specify both files and --all"
where
ww = WarnUnmatchLsFiles
start :: LogOptions -> (FilePath -> Outputter) -> RawFilePath -> Key -> CommandStart
start o outputter file key = do

View file

@ -31,7 +31,7 @@ run _ file = seekSingleGitFile file >>= \case
-- But, this plumbing command does not recurse through directories.
seekSingleGitFile :: FilePath -> Annex (Maybe RawFilePath)
seekSingleGitFile file = do
(l, cleanup) <- inRepo (Git.LsFiles.inRepo [toRawFilePath file])
(l, cleanup) <- inRepo (Git.LsFiles.inRepo [] [toRawFilePath file])
r <- case l of
(f:[]) | takeFileName (fromRawFilePath f) == takeFileName file ->
return (Just f)

View file

@ -75,15 +75,16 @@ seek :: MetaDataOptions -> CommandSeek
seek o = case batchOption o of
NoBatch -> do
c <- liftIO currentVectorClock
let ww = WarnUnmatchLsFiles
let seeker = case getSet o of
Get _ -> withFilesInGit
GetAll -> withFilesInGit
Set _ -> withFilesInGitNonRecursive
Get _ -> withFilesInGit ww
GetAll -> withFilesInGit ww
Set _ -> withFilesInGitNonRecursive ww
"Not recursively setting metadata. Use --force to do that."
withKeyOptions (keyOptions o) False
(commandAction . startKeys c o)
(seeker (commandAction . (whenAnnexed (start c o))))
=<< workTreeItems (forFiles o)
=<< workTreeItems ww (forFiles o)
Batch fmt -> withMessageState $ \s -> case outputType s of
JSONOutput _ -> ifM limited
( giveup "combining --batch with file matching options is not currently supported"

View file

@ -26,7 +26,10 @@ cmd = withGlobalOptions [annexedMatchingOptions] $
paramPaths (withParams seek)
seek :: CmdParams -> CommandSeek
seek = withFilesInGit (commandAction . (whenAnnexed start)) <=< workTreeItems
seek = withFilesInGit ww (commandAction . (whenAnnexed start))
<=< workTreeItems ww
where
ww = WarnUnmatchLsFiles
start :: RawFilePath -> Key -> CommandStart
start file key = do

View file

@ -44,12 +44,13 @@ seek :: MirrorOptions -> CommandSeek
seek o = startConcurrency stages $
withKeyOptions (keyOptions o) False
(commandAction . startKey o (AssociatedFile Nothing))
(withFilesInGit (commandAction . (whenAnnexed $ start o)))
=<< workTreeItems (mirrorFiles o)
(withFilesInGit ww (commandAction . (whenAnnexed $ start o)))
=<< workTreeItems ww (mirrorFiles o)
where
stages = case fromToOptions o of
FromRemote _ -> downloadStages
ToRemote _ -> commandStages
ww = WarnUnmatchLsFiles
start :: MirrorOptions -> RawFilePath -> Key -> CommandStart
start o file k = startKey o afile (k, ai)

View file

@ -60,13 +60,14 @@ seek o = startConcurrency stages $ do
Batch fmt -> batchFilesMatching fmt (go . toRawFilePath)
NoBatch -> withKeyOptions (keyOptions o) False
(commandAction . startKey (fromToOptions o) (removeWhen o))
(withFilesInGit (commandAction . go))
=<< workTreeItems (moveFiles o)
(withFilesInGit ww (commandAction . go))
=<< workTreeItems ww (moveFiles o)
where
stages = case fromToOptions o of
Right (FromRemote _) -> downloadStages
Right (ToRemote _) -> commandStages
Left ToHere -> downloadStages
ww = WarnUnmatchLsFiles
start :: FromToHereOptions -> RemoveWhen -> RawFilePath -> Key -> CommandStart
start fromto removewhen f k = start' fromto removewhen afile k ai

View file

@ -129,7 +129,9 @@ send ups fs = do
-- expensive.
starting "sending files" (ActionItemOther Nothing) $
withTmpFile "send" $ \t h -> do
fs' <- seekHelper LsFiles.inRepo =<< workTreeItems fs
let ww = WarnUnmatchLsFiles
fs' <- seekHelper ww LsFiles.inRepo
=<< workTreeItems ww fs
matcher <- Limit.getMatcher
let addlist f o = whenM (matcher $ MatchingFile $ FileInfo f f) $
liftIO $ hPutStrLn h o

View file

@ -32,7 +32,8 @@ cmd = command "pre-commit" SectionPlumbing
seek :: CmdParams -> CommandSeek
seek ps = do
l <- workTreeItems ps
let ww = WarnUnmatchWorkTreeItems
l <- workTreeItems ww ps
-- fix symlinks to files being committed
flip withFilesToBeCommitted l $ \f -> commandAction $
maybe stop (Command.Fix.start Command.Fix.FixSymlinks f)

View file

@ -636,11 +636,11 @@ seekSyncContent o rs currbranch = do
Just WantAllKeys -> Just <$> genBloomFilter (seekworktree mvar [])
_ -> case currbranch of
(Just origbranch, Just adj) | adjustmentHidesFiles adj -> do
l <- workTreeItems' (AllowHidden True) (contentOfOption o)
l <- workTreeItems' (AllowHidden True) ww (contentOfOption o)
seekincludinghidden origbranch mvar l (const noop)
pure Nothing
_ -> do
l <- workTreeItems (contentOfOption o)
l <- workTreeItems ww (contentOfOption o)
seekworktree mvar l (const noop)
pure Nothing
withKeyOptions' (keyOptions o) False
@ -651,13 +651,15 @@ seekSyncContent o rs currbranch = do
liftIO $ not <$> isEmptyMVar mvar
where
seekworktree mvar l bloomfeeder =
seekHelper LsFiles.inRepo l
seekHelper ww LsFiles.inRepo l
>>= gofiles bloomfeeder mvar
seekincludinghidden origbranch mvar l bloomfeeder =
seekHelper (LsFiles.inRepoOrBranch origbranch) l
seekHelper ww (LsFiles.inRepoOrBranch origbranch) l
>>= gofiles bloomfeeder mvar
ww = WarnUnmatchLsFiles
gofiles bloomfeeder mvar = mapM_ $ \f ->
ifAnnexed f
(go (Right bloomfeeder) mvar (AssociatedFile (Just f)))

View file

@ -23,7 +23,10 @@ cmd = withGlobalOptions [annexedMatchingOptions] $
paramPaths (withParams seek)
seek :: CmdParams -> CommandSeek
seek ps = (withFilesInGit $ commandAction . whenAnnexed start) =<< workTreeItems ps
seek ps = (withFilesInGit ww $ commandAction . whenAnnexed start)
=<< workTreeItems ww ps
where
ww = WarnUnmatchLsFiles
start :: RawFilePath -> Key -> CommandStart
start file key = stopUnless (inAnnex key) $

View file

@ -27,7 +27,7 @@ seek :: CmdParams -> CommandSeek
seek ps = do
-- Safety first; avoid any undo that would touch files that are not
-- in the index.
(fs, cleanup) <- inRepo $ LsFiles.notInRepo False (map toRawFilePath ps)
(fs, cleanup) <- inRepo $ LsFiles.notInRepo [] False (map toRawFilePath ps)
unless (null fs) $
giveup $ "Cannot undo changes to files that are not checked into git: " ++ unwords (map fromRawFilePath fs)
void $ liftIO $ cleanup

View file

@ -41,11 +41,13 @@ check = do
seek :: CmdParams -> CommandSeek
seek ps = do
l <- workTreeItems ps
l <- workTreeItems ww ps
withFilesNotInGit (commandAction . whenAnnexed (startCheckIncomplete . fromRawFilePath)) l
Annex.changeState $ \s -> s { Annex.fast = True }
withFilesInGit (commandAction . whenAnnexed Command.Unannex.start) l
withFilesInGit ww (commandAction . whenAnnexed Command.Unannex.start) l
finish
where
ww = WarnUnmatchLsFiles
{- git annex symlinks that are not checked into git could be left by an
- interrupted add. -}

View file

@ -27,7 +27,10 @@ mkcmd n d = withGlobalOptions [jsonOptions, annexedMatchingOptions] $
command n SectionCommon d paramPaths (withParams seek)
seek :: CmdParams -> CommandSeek
seek ps = withFilesInGit (commandAction . whenAnnexed start) =<< workTreeItems ps
seek ps = withFilesInGit ww (commandAction . whenAnnexed start)
=<< workTreeItems ww ps
where
ww = WarnUnmatchLsFiles
start :: RawFilePath -> Key -> CommandStart
start file key = ifM (isJust <$> isAnnexLink file)

View file

@ -210,9 +210,9 @@ withKeysReferenced' mdir initial a = do
( return ([], return True)
, do
top <- fromRepo Git.repoPath
inRepo $ LsFiles.allFiles [top]
inRepo $ LsFiles.allFiles [] [top]
)
Just dir -> inRepo $ LsFiles.inRepo [toRawFilePath dir]
Just dir -> inRepo $ LsFiles.inRepo [] [toRawFilePath dir]
go v [] = return v
go v (f:fs) = do
mk <- lookupFile f

View file

@ -101,7 +101,7 @@ checkoutViewBranch view mkbranch = do
- removed.) -}
top <- liftIO . absPath . fromRawFilePath =<< fromRepo Git.repoPath
(l, cleanup) <- inRepo $
LsFiles.notInRepoIncludingEmptyDirectories False
LsFiles.notInRepoIncludingEmptyDirectories [] False
[toRawFilePath top]
forM_ l (removeemptydir top)
liftIO $ void cleanup

View file

@ -57,8 +57,10 @@ seek o = do
NoBatch ->
withKeyOptions (keyOptions o) False
(commandAction . startKeys o m)
(withFilesInGit (commandAction . go))
=<< workTreeItems (whereisFiles o)
(withFilesInGit ww (commandAction . go))
=<< workTreeItems ww (whereisFiles o)
where
ww = WarnUnmatchLsFiles
start :: WhereisOptions -> M.Map UUID Remote -> RawFilePath -> Key -> CommandStart
start o remotemap file key =