wip RawFilePath 2x git-annex find speedup

Finally builds (oh the agoncy of making it build), but still very
unmergable, only Command.Find is included and lots of stuff is badly
hacked to make it compile.

Benchmarking vs master, this git-annex find is significantly faster!
Specifically:

	num files	old	new	speedup
	48500		4.77	3.73	28%
	12500		1.36	1.02	66%
	20		0.075	0.074	0% (so startup time is unchanged)

That's without really finishing the optimization. Things still to do:

* Eliminate all the fromRawFilePath, toRawFilePath, encodeBS,
  decodeBS conversions.
* Use versions of IO actions like getFileStatus that take a RawFilePath.
* Eliminate some Data.ByteString.Lazy.toStrict, which is a slow copy.
* Use ByteString for parsing git config to speed up startup.

It's likely several of those will speed up git-annex find further.
And other commands will certianly benefit even more.
This commit is contained in:
Joey Hess 2019-11-26 15:27:22 -04:00
parent 6a97ff6b3a
commit 067aabdd48
No known key found for this signature in database
GPG key ID: DB12DB0FF05F8F38
61 changed files with 380 additions and 296 deletions

View file

@ -128,28 +128,28 @@ addExportedLocation :: ExportHandle -> Key -> ExportLocation -> IO ()
addExportedLocation h k el = queueDb h $ do
void $ insertUnique $ Exported ik ef
let edirs = map
(\ed -> ExportedDirectory (toSFilePath (fromExportDirectory ed)) ef)
(\ed -> ExportedDirectory (toSFilePath (fromRawFilePath (fromExportDirectory ed))) ef)
(exportDirectories el)
putMany edirs
where
ik = toIKey k
ef = toSFilePath (fromExportLocation el)
ef = toSFilePath $ fromRawFilePath $ fromExportLocation el
removeExportedLocation :: ExportHandle -> Key -> ExportLocation -> IO ()
removeExportedLocation h k el = queueDb h $ do
deleteWhere [ExportedKey ==. ik, ExportedFile ==. ef]
let subdirs = map (toSFilePath . fromExportDirectory)
let subdirs = map (toSFilePath . fromRawFilePath . fromExportDirectory)
(exportDirectories el)
deleteWhere [ExportedDirectoryFile ==. ef, ExportedDirectorySubdir <-. subdirs]
where
ik = toIKey k
ef = toSFilePath (fromExportLocation el)
ef = toSFilePath $ fromRawFilePath $ fromExportLocation el
{- Note that this does not see recently queued changes. -}
getExportedLocation :: ExportHandle -> Key -> IO [ExportLocation]
getExportedLocation (ExportHandle h _) k = H.queryDbQueue h $ do
l <- selectList [ExportedKey ==. ik] []
return $ map (mkExportLocation . fromSFilePath . exportedFile . entityVal) l
return $ map (mkExportLocation . toRawFilePath . fromSFilePath . exportedFile . entityVal) l
where
ik = toIKey k
@ -159,13 +159,13 @@ isExportDirectoryEmpty (ExportHandle h _) d = H.queryDbQueue h $ do
l <- selectList [ExportedDirectorySubdir ==. ed] []
return $ null l
where
ed = toSFilePath $ fromExportDirectory d
ed = toSFilePath $ fromRawFilePath $ fromExportDirectory d
{- Get locations in the export that might contain a key. -}
getExportTree :: ExportHandle -> Key -> IO [ExportLocation]
getExportTree (ExportHandle h _) k = H.queryDbQueue h $ do
l <- selectList [ExportTreeKey ==. ik] []
return $ map (mkExportLocation . fromSFilePath . exportTreeFile . entityVal) l
return $ map (mkExportLocation . toRawFilePath . fromSFilePath . exportTreeFile . entityVal) l
where
ik = toIKey k
@ -181,21 +181,21 @@ getExportTreeKey (ExportHandle h _) el = H.queryDbQueue h $ do
map (fromIKey . exportTreeKey . entityVal)
<$> selectList [ExportTreeFile ==. ef] []
where
ef = toSFilePath (fromExportLocation el)
ef = toSFilePath (fromRawFilePath $ fromExportLocation el)
addExportTree :: ExportHandle -> Key -> ExportLocation -> IO ()
addExportTree h k loc = queueDb h $
void $ insertUnique $ ExportTree ik ef
where
ik = toIKey k
ef = toSFilePath (fromExportLocation loc)
ef = toSFilePath (fromRawFilePath $ fromExportLocation loc)
removeExportTree :: ExportHandle -> Key -> ExportLocation -> IO ()
removeExportTree h k loc = queueDb h $
deleteWhere [ExportTreeKey ==. ik, ExportTreeFile ==. ef]
where
ik = toIKey k
ef = toSFilePath (fromExportLocation loc)
ef = toSFilePath (fromRawFilePath $ fromExportLocation loc)
-- An action that is passed the old and new values that were exported,
-- and updates state.
@ -220,7 +220,7 @@ mkExportDiffUpdater removeold addnew h srcek dstek i = do
Nothing -> return ()
Just k -> liftIO $ addnew h (asKey k) loc
where
loc = mkExportLocation $ getTopFilePath $ Git.DiffTree.file i
loc = mkExportLocation $ toRawFilePath $ getTopFilePath $ Git.DiffTree.file i
runExportDiffUpdater :: ExportDiffUpdater -> ExportHandle -> Sha -> Sha -> Annex ()
runExportDiffUpdater updater h old new = do