mirrors/git-annex
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

wip RawFilePath 2x git-annex find speedup

Browse source 
Finally builds (oh the agoncy of making it build), but still very
unmergable, only Command.Find is included and lots of stuff is badly
hacked to make it compile.

Benchmarking vs master, this git-annex find is significantly faster!
Specifically:

	num files	old	new	speedup
	48500		4.77	3.73	28%
	12500		1.36	1.02	66%
	20		0.075	0.074	0% (so startup time is unchanged)

That's without really finishing the optimization. Things still to do:

* Eliminate all the fromRawFilePath, toRawFilePath, encodeBS,
  decodeBS conversions.
* Use versions of IO actions like getFileStatus that take a RawFilePath.
* Eliminate some Data.ByteString.Lazy.toStrict, which is a slow copy.
* Use ByteString for parsing git config to speed up startup.

It's likely several of those will speed up git-annex find further.
And other commands will certianly benefit even more.
This commit is contained in:
Joey Hess 2019-11-26 15:27:22 -04:00
parent 6a97ff6b3a
commit 067aabdd48
No known key found for this signature in database
GPG key ID: DB12DB0FF05F8F38
61 changed files with 380 additions and 296 deletions
Download patch file Download diff file Expand all files Collapse all files

18
Types/ActionItem.hs
View file

@ -12,15 +12,17 @@ module Types.ActionItem where
import Key
import Types.Transfer
import Git.FilePath
import Utility.FileSystemEncoding
import Data.Maybe
import qualified Data.ByteString as S
data ActionItem
= ActionItemAssociatedFile AssociatedFile Key
| ActionItemKey Key
| ActionItemBranchFilePath BranchFilePath Key
| ActionItemFailedTransfer Transfer TransferInfo
| ActionItemWorkTreeFile FilePath
| ActionItemWorkTreeFile RawFilePath
| ActionItemOther (Maybe String)
-- Use to avoid more than one thread concurrently processing the
-- same Key.
@ -39,10 +41,10 @@ instance MkActionItem (AssociatedFile, Key) where
instance MkActionItem (Key, AssociatedFile) where
mkActionItem = uncurry $ flip ActionItemAssociatedFile
instance MkActionItem (Key, FilePath) where
instance MkActionItem (Key, RawFilePath) where
mkActionItem (key, file) = ActionItemAssociatedFile (AssociatedFile (Just file)) key
instance MkActionItem (FilePath, Key) where
instance MkActionItem (RawFilePath, Key) where
mkActionItem (file, key) = mkActionItem (key, file)
instance MkActionItem Key where
@ -54,16 +56,16 @@ instance MkActionItem (BranchFilePath, Key) where
instance MkActionItem (Transfer, TransferInfo) where
mkActionItem = uncurry ActionItemFailedTransfer
actionItemDesc :: ActionItem -> String
actionItemDesc :: ActionItem -> S.ByteString
actionItemDesc (ActionItemAssociatedFile (AssociatedFile (Just f)) _) = f
actionItemDesc (ActionItemAssociatedFile (AssociatedFile Nothing) k) =
serializeKey k
actionItemDesc (ActionItemKey k) = serializeKey k
serializeKey' k
actionItemDesc (ActionItemKey k) = serializeKey' k
actionItemDesc (ActionItemBranchFilePath bfp _) = descBranchFilePath bfp
actionItemDesc (ActionItemFailedTransfer t i) = actionItemDesc $
ActionItemAssociatedFile (associatedFile i) (transferKey t)
actionItemDesc (ActionItemWorkTreeFile f) = f
actionItemDesc (ActionItemOther s) = fromMaybe "" s
actionItemDesc (ActionItemOther s) = encodeBS' (fromMaybe "" s)
actionItemDesc (OnlyActionOn _ ai) = actionItemDesc ai
actionItemKey :: ActionItem -> Maybe Key
@ -75,7 +77,7 @@ actionItemKey (ActionItemWorkTreeFile _) = Nothing
actionItemKey (ActionItemOther _) = Nothing
actionItemKey (OnlyActionOn _ ai) = actionItemKey ai
actionItemWorkTreeFile :: ActionItem -> Maybe FilePath
actionItemWorkTreeFile :: ActionItem -> Maybe RawFilePath
actionItemWorkTreeFile (ActionItemAssociatedFile (AssociatedFile af) _) = af
actionItemWorkTreeFile (ActionItemWorkTreeFile f) = Just f
actionItemWorkTreeFile (OnlyActionOn _ ai) = actionItemWorkTreeFile ai

3
Types/Key.hs
View file

@ -36,6 +36,7 @@ import Data.ByteString.Builder
import Data.ByteString.Builder.Extra
import qualified Data.Attoparsec.ByteString as A
import qualified Data.Attoparsec.ByteString.Char8 as A8
import Utility.FileSystemEncoding
import Data.List
import System.Posix.Types
import Foreign.C.Types
@ -200,7 +201,7 @@ splitKeyNameExtension' :: S.ByteString -> (S.ByteString, S.ByteString)
splitKeyNameExtension' keyname = S8.span (/= '.') keyname
{- A filename may be associated with a Key. -}
newtype AssociatedFile = AssociatedFile (Maybe FilePath)
newtype AssociatedFile = AssociatedFile (Maybe RawFilePath)
deriving (Show, Eq, Ord)
{- There are several different varieties of keys. -}

6
Types/Transfer.hs
View file

@ -15,6 +15,7 @@ import Types.Key
import Utility.PID
import Utility.QuickCheck
import Utility.Url
import Utility.FileSystemEncoding
import Data.Time.Clock.POSIX
import Control.Concurrent
@ -71,8 +72,7 @@ instance Arbitrary TransferInfo where
<*> pure Nothing -- cannot generate a ThreadID
<*> pure Nothing -- remote not needed
<*> arbitrary
-- associated file cannot be empty (but can be Nothing)
<*> (AssociatedFile <$> arbitrary `suchThat` (/= Just ""))
<*> arbitrary
<*> arbitrary
class Observable a where
@ -101,7 +101,7 @@ class Transferrable t where
descTransfrerrable :: t -> Maybe String
instance Transferrable AssociatedFile where
descTransfrerrable (AssociatedFile af) = af
descTransfrerrable (AssociatedFile af) = fromRawFilePath <$> af
instance Transferrable URLString where
descTransfrerrable = Just