mirrors/git-annex
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

convert Key to ShortByteString

Browse source 
This adds the overhead of a copy when serializing and deserializing keys.
I have not benchmarked much, but runtimes seem barely changed at all by that.

When a lot of keys are in memory, it improves memory use.

And, it prevents keys sometimes getting PINNED in memory and failing to GC,
which is a problem ByteString has sometimes. In particular, git-annex sync
from a borg special remote had that problem and this improved its memory
use by a large amount.

Sponsored-by: Shae Erisson on Patreon
This commit is contained in:
Joey Hess 2021-10-05 20:20:08 -04:00
parent 012b71e471
commit 19e78816f0
No known key found for this signature in database
GPG key ID: DB12DB0FF05F8F38
15 changed files with 65 additions and 36 deletions
Download patch file Download diff file Expand all files Collapse all files

13
Backend/Hash.hs
View file

@ -24,6 +24,7 @@ import Utility.Metered
import qualified Utility.RawFilePath as R
import qualified Data.ByteString as S
import qualified Data.ByteString.Short as S (toShort, fromShort)
import qualified Data.ByteString.Char8 as S8
import qualified Data.ByteString.Lazy as L
import Control.DeepSeq
@ -106,7 +107,7 @@ keyValue hash source meterupdate = do
filesize <- liftIO $ getFileSize file
s <- hashFile hash file meterupdate
return $ mkKey $ \k -> k
{ keyName = encodeBS s
{ keyName = S.toShort (encodeBS s)
, keyVariety = hashKeyVariety hash (HasExt False)
, keySize = Just filesize
}
@ -160,7 +161,7 @@ needsUpgrade :: Key -> Bool
needsUpgrade key = or
[ "\\" `S8.isPrefixOf` keyHash key
, S.any (not . validInExtension) (snd $ splitKeyNameExtension key)
, not (hasExt (fromKey keyVariety key)) && keyHash key /= fromKey keyName key
, not (hasExt (fromKey keyVariety key)) && keyHash key /= S.fromShort (fromKey keyName key)
]
trivialMigrate :: Key -> Backend -> AssociatedFile -> Annex (Maybe Key)
@ -171,14 +172,14 @@ trivialMigrate' :: Key -> Backend -> AssociatedFile -> Maybe Int -> Maybe Key
trivialMigrate' oldkey newbackend afile maxextlen
{- Fast migration from hashE to hash backend. -}
| migratable && hasExt oldvariety = Just $ alterKey oldkey $ \d -> d
{ keyName = keyHash oldkey
{ keyName = S.toShort (keyHash oldkey)
, keyVariety = newvariety
}
{- Fast migration from hash to hashE backend. -}
| migratable && hasExt newvariety = case afile of
AssociatedFile Nothing -> Nothing
AssociatedFile (Just file) -> Just $ alterKey oldkey $ \d -> d
{ keyName = keyHash oldkey
{ keyName = S.toShort $ keyHash oldkey
<> selectExtension maxextlen file
, keyVariety = newvariety
}
@ -186,9 +187,9 @@ trivialMigrate' oldkey newbackend afile maxextlen
- non-extension preserving key, with an extension
- in its keyName. -}
| newvariety == oldvariety && not (hasExt oldvariety) &&
keyHash oldkey /= fromKey keyName oldkey =
keyHash oldkey /= S.fromShort (fromKey keyName oldkey) =
Just $ alterKey oldkey $ \d -> d
{ keyName = keyHash oldkey
{ keyName = S.toShort (keyHash oldkey)
}
| otherwise = Nothing
where