mirrors/git-annex
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

cache the serialization of a Key

Browse source 
This will speed up the common case where a Key is deserialized from
disk, but is then serialized to build eg, the path to the annex object.

Previously attempted in 4536c93bb2
and reverted in 96aba8eff7.
The problems mentioned in the latter commit are addressed now:

Read/Show of KeyData is backwards-compatible with Read/Show of Key from before
this change, so Types.Distribution will keep working.

The Eq instance is fixed.

Also, Key has smart constructors, avoiding needing to remember to update
the cached serialization.

Used git-annex benchmark:
  find is 7% faster
  whereis is 3% faster
  get when all files are already present is 5% faster
Generally, the benchmarks are running 0.1 seconds faster per 2000 files,
on a ram disk in my laptop.
This commit is contained in:
Joey Hess 2019-11-22 16:24:04 -04:00
parent e296637737
commit 81d402216d
53 changed files with 388 additions and 289 deletions
Download patch file Download diff file Expand all files Collapse all files

25
Backend/Hash.hs
View file

@ -91,7 +91,7 @@ keyValue hash source meterupdate = do
let file = contentLocation source
filesize <- liftIO $ getFileSize file
s <- hashFile hash file meterupdate
return $ Just $ stubKey
return $ Just $ mkKey $ \k -> k
{ keyName = encodeBS s
, keyVariety = hashKeyVariety hash (HasExt False)
, keySize = Just filesize
@ -105,8 +105,8 @@ keyValueE hash source meterupdate =
addE k = do
maxlen <- annexMaxExtensionLength <$> Annex.getGitConfig
let ext = selectExtension maxlen (keyFilename source)
return $ Just $ k
{ keyName = keyName k <> encodeBS ext
return $ Just $ alterKey k $ \d -> d
{ keyName = keyName d <> encodeBS ext
, keyVariety = hashKeyVariety hash (HasExt True)
}
@ -169,7 +169,7 @@ needsUpgrade :: Key -> Bool
needsUpgrade key = or
[ "\\" `S8.isPrefixOf` keyHash key
, any (not . validInExtension) (decodeBS $ snd $ splitKeyNameExtension key)
, not (hasExt (keyVariety key)) && keyHash key /= keyName key
, not (hasExt (fromKey keyVariety key)) && keyHash key /= fromKey keyName key
]
trivialMigrate :: Key -> Backend -> AssociatedFile -> Annex (Maybe Key)
@ -179,14 +179,14 @@ trivialMigrate oldkey newbackend afile = trivialMigrate' oldkey newbackend afile
trivialMigrate' :: Key -> Backend -> AssociatedFile -> Maybe Int -> Maybe Key
trivialMigrate' oldkey newbackend afile maxextlen
{- Fast migration from hashE to hash backend. -}
| migratable && hasExt oldvariety = Just $ oldkey
| migratable && hasExt oldvariety = Just $ alterKey oldkey $ \d -> d
{ keyName = keyHash oldkey
, keyVariety = newvariety
}
{- Fast migration from hash to hashE backend. -}
| migratable && hasExt newvariety = case afile of
AssociatedFile Nothing -> Nothing
AssociatedFile (Just file) -> Just $ oldkey
AssociatedFile (Just file) -> Just $ alterKey oldkey $ \d -> d
{ keyName = keyHash oldkey
<> encodeBS (selectExtension maxextlen file)
, keyVariety = newvariety
@ -195,14 +195,15 @@ trivialMigrate' oldkey newbackend afile maxextlen
- non-extension preserving key, with an extension
- in its keyName. -}
| newvariety == oldvariety && not (hasExt oldvariety) &&
keyHash oldkey /= keyName oldkey = Just $ oldkey
{ keyName = keyHash oldkey
}
keyHash oldkey /= fromKey keyName oldkey =
Just $ alterKey oldkey $ \d -> d
{ keyName = keyHash oldkey
}
| otherwise = Nothing
where
migratable = oldvariety /= newvariety
&& sameExceptExt oldvariety newvariety
oldvariety = keyVariety oldkey
oldvariety = fromKey keyVariety oldkey
newvariety = backendVariety newbackend
hashFile :: Hash -> FilePath -> MeterUpdate -> Annex String
@ -294,5 +295,7 @@ testKeyBackend =
let b = genBackendE (SHA2Hash (HashSize 256))
in b { getKey = \ks p -> (fmap addE) <$> getKey b ks p }
where
addE k = k { keyName = keyName k <> longext }
addE k = alterKey k $ \d -> d
{ keyName = keyName d <> longext
}
longext = ".this-is-a-test-key"