convert Key to ShortByteString

This adds the overhead of a copy when serializing and deserializing keys.
I have not benchmarked much, but runtimes seem barely changed at all by that.

When a lot of keys are in memory, it improves memory use.

And, it prevents keys sometimes getting PINNED in memory and failing to GC,
which is a problem ByteString has sometimes. In particular, git-annex sync
from a borg special remote had that problem and this improved its memory
use by a large amount.

Sponsored-by: Shae Erisson on Patreon
This commit is contained in:
Joey Hess 2021-10-05 20:20:08 -04:00
parent 012b71e471
commit 19e78816f0
No known key found for this signature in database
GPG key ID: DB12DB0FF05F8F38
15 changed files with 65 additions and 36 deletions

View file

@ -29,6 +29,7 @@ module Types.Key (
) where
import qualified Data.ByteString as S
import qualified Data.ByteString.Short as S (ShortByteString, toShort, fromShort)
import qualified Data.ByteString.Char8 as S8
import qualified Data.ByteString.Lazy as L
import Data.ByteString.Builder
@ -49,7 +50,7 @@ import Prelude
{- A Key has a unique name, which is derived from a particular backend,
- and may contain other optional metadata. -}
data KeyData = Key
{ keyName :: S.ByteString
{ keyName :: S.ShortByteString
, keyVariety :: KeyVariety
, keySize :: Maybe Integer
, keyMtime :: Maybe EpochTime
@ -66,7 +67,7 @@ instance NFData KeyData
-}
data Key = MkKey
{ keyData :: KeyData
, keySerialization :: S.ByteString
, keySerialization :: S.ShortByteString
} deriving (Show, Generic)
instance Eq Key where
@ -111,8 +112,8 @@ isKeyPrefix s = [fieldSep, fieldSep] `isInfixOf` s
fieldSep :: Char
fieldSep = '-'
mkKeySerialization :: KeyData -> S.ByteString
mkKeySerialization = L.toStrict
mkKeySerialization :: KeyData -> S.ShortByteString
mkKeySerialization = S.toShort . L.toStrict
. toLazyByteStringWith (safeStrategy 128 smallChunkSize) L.empty
. buildKeyData
@ -127,7 +128,7 @@ buildKeyData k = byteString (formatKeyVariety (keyVariety k))
<> 'm' ?: (integerDec . (\(CTime t) -> fromIntegral t) <$> keyMtime k)
<> 'S' ?: (integerDec <$> keyChunkSize k)
<> 'C' ?: (integerDec <$> keyChunkNum k)
<> sepbefore (sepbefore (byteString (keyName k)))
<> sepbefore (sepbefore (byteString (S.fromShort (keyName k))))
where
sepbefore s = char7 fieldSep <> s
c ?: (Just b) = sepbefore (char7 c <> b)
@ -156,7 +157,7 @@ keyParser = do
if validKeyName v n
then
let d = Key
{ keyName = n
{ keyName = S.toShort n
, keyVariety = v
, keySize = s
, keyMtime = m
@ -195,7 +196,7 @@ validKeyName kv name
- keyName minus extension, and the extension (including leading dot).
-}
splitKeyNameExtension :: Key -> (S.ByteString, S.ByteString)
splitKeyNameExtension = splitKeyNameExtension' . keyName . keyData
splitKeyNameExtension = splitKeyNameExtension' . S.fromShort . keyName . keyData
splitKeyNameExtension' :: S.ByteString -> (S.ByteString, S.ByteString)
splitKeyNameExtension' keyname = S8.span (/= '.') keyname