optimisation: stream keys through git cat-file --buffer

This is only implemented for git-annex get so far. It makes git-annex
get nearly twice as fast in a repo with 10k files, all of them present!

But, see the TODO for some caveats.
This commit is contained in:
Joey Hess 2020-07-10 13:54:52 -04:00
parent 1df9e72a78
commit 4c9ad1de46
No known key found for this signature in database
GPG key ID: DB12DB0FF05F8F38
6 changed files with 78 additions and 39 deletions

View file

@ -200,7 +200,7 @@ changeExport r db (PreferredFiltered new) = do
mapdiff a oldtreesha newtreesha = do
(diff, cleanup) <- inRepo $
Git.DiffTree.diffTreeRecursive oldtreesha newtreesha
seekActions $ pure $ map a diff
sequence_ $ map a diff
void $ liftIO cleanup
-- Map of old and new filenames for each changed ExportKey in a diff.

View file

@ -40,12 +40,13 @@ optParser desc = GetOptions
seek :: GetOptions -> CommandSeek
seek o = startConcurrency downloadStages $ do
from <- maybe (pure Nothing) (Just <$$> getParsed) (getFrom o)
let go = whenAnnexed $ start o from
let go = start o from
case batchOption o of
Batch fmt -> batchFilesMatching fmt (go . toRawFilePath)
Batch fmt -> batchFilesMatching fmt
(whenAnnexed go . toRawFilePath)
NoBatch -> withKeyOptions (keyOptions o) (autoMode o)
(commandAction . startKeys from)
(withFilesInGit ww (commandAction . go))
(withFilesInGitAnnex ww (\f k -> commandAction (go f k)))
=<< workTreeItems ww (getFiles o)
where
ww = WarnUnmatchLsFiles

View file

@ -130,7 +130,7 @@ send ups fs = do
starting "sending files" (ActionItemOther Nothing) $
withTmpFile "send" $ \t h -> do
let ww = WarnUnmatchLsFiles
fs' <- seekHelper ww LsFiles.inRepo
fs' <- seekHelper id ww LsFiles.inRepo
=<< workTreeItems ww fs
matcher <- Limit.getMatcher
let addlist f o = whenM (matcher $ MatchingFile $ FileInfo f f) $

View file

@ -652,11 +652,11 @@ seekSyncContent o rs currbranch = do
liftIO $ not <$> isEmptyMVar mvar
where
seekworktree mvar l bloomfeeder =
seekHelper ww LsFiles.inRepo l
seekHelper id ww LsFiles.inRepo l
>>= gofiles bloomfeeder mvar
seekincludinghidden origbranch mvar l bloomfeeder =
seekHelper ww (LsFiles.inRepoOrBranch origbranch) l
seekHelper id ww (LsFiles.inRepoOrBranch origbranch) l
>>= gofiles bloomfeeder mvar
ww = WarnUnmatchLsFiles