convert Key to ShortByteString
This adds the overhead of a copy when serializing and deserializing keys. I have not benchmarked much, but runtimes seem barely changed at all by that. When a lot of keys are in memory, it improves memory use. And, it prevents keys sometimes getting PINNED in memory and failing to GC, which is a problem ByteString has sometimes. In particular, git-annex sync from a borg special remote had that problem and this improved its memory use by a large amount. Sponsored-by: Shae Erisson on Patreon
This commit is contained in:
parent
012b71e471
commit
19e78816f0
15 changed files with 65 additions and 36 deletions
|
@ -20,6 +20,7 @@ import Utility.Metered
|
|||
import qualified Utility.SimpleProtocol as Proto
|
||||
|
||||
import qualified Data.ByteString as S
|
||||
import qualified Data.ByteString.Short as S (toShort, fromShort)
|
||||
import qualified Data.Map.Strict as M
|
||||
import Data.Char
|
||||
import Control.Concurrent
|
||||
|
@ -285,7 +286,7 @@ toProtoKey k = ProtoKey $ alterKey k $ \d -> d
|
|||
-- The extension can be easily removed, because the protocol
|
||||
-- documentation does not allow '.' to be used in the keyName,
|
||||
-- so the first one is the extension.
|
||||
{ keyName = S.takeWhile (/= dot) (keyName d)
|
||||
{ keyName = S.toShort (S.takeWhile (/= dot) (S.fromShort (keyName d)))
|
||||
, keyVariety = setHasExt (HasExt False) (keyVariety d)
|
||||
}
|
||||
where
|
||||
|
|
|
@ -24,6 +24,7 @@ import Utility.Metered
|
|||
import qualified Utility.RawFilePath as R
|
||||
|
||||
import qualified Data.ByteString as S
|
||||
import qualified Data.ByteString.Short as S (toShort, fromShort)
|
||||
import qualified Data.ByteString.Char8 as S8
|
||||
import qualified Data.ByteString.Lazy as L
|
||||
import Control.DeepSeq
|
||||
|
@ -106,7 +107,7 @@ keyValue hash source meterupdate = do
|
|||
filesize <- liftIO $ getFileSize file
|
||||
s <- hashFile hash file meterupdate
|
||||
return $ mkKey $ \k -> k
|
||||
{ keyName = encodeBS s
|
||||
{ keyName = S.toShort (encodeBS s)
|
||||
, keyVariety = hashKeyVariety hash (HasExt False)
|
||||
, keySize = Just filesize
|
||||
}
|
||||
|
@ -160,7 +161,7 @@ needsUpgrade :: Key -> Bool
|
|||
needsUpgrade key = or
|
||||
[ "\\" `S8.isPrefixOf` keyHash key
|
||||
, S.any (not . validInExtension) (snd $ splitKeyNameExtension key)
|
||||
, not (hasExt (fromKey keyVariety key)) && keyHash key /= fromKey keyName key
|
||||
, not (hasExt (fromKey keyVariety key)) && keyHash key /= S.fromShort (fromKey keyName key)
|
||||
]
|
||||
|
||||
trivialMigrate :: Key -> Backend -> AssociatedFile -> Annex (Maybe Key)
|
||||
|
@ -171,14 +172,14 @@ trivialMigrate' :: Key -> Backend -> AssociatedFile -> Maybe Int -> Maybe Key
|
|||
trivialMigrate' oldkey newbackend afile maxextlen
|
||||
{- Fast migration from hashE to hash backend. -}
|
||||
| migratable && hasExt oldvariety = Just $ alterKey oldkey $ \d -> d
|
||||
{ keyName = keyHash oldkey
|
||||
{ keyName = S.toShort (keyHash oldkey)
|
||||
, keyVariety = newvariety
|
||||
}
|
||||
{- Fast migration from hash to hashE backend. -}
|
||||
| migratable && hasExt newvariety = case afile of
|
||||
AssociatedFile Nothing -> Nothing
|
||||
AssociatedFile (Just file) -> Just $ alterKey oldkey $ \d -> d
|
||||
{ keyName = keyHash oldkey
|
||||
{ keyName = S.toShort $ keyHash oldkey
|
||||
<> selectExtension maxextlen file
|
||||
, keyVariety = newvariety
|
||||
}
|
||||
|
@ -186,9 +187,9 @@ trivialMigrate' oldkey newbackend afile maxextlen
|
|||
- non-extension preserving key, with an extension
|
||||
- in its keyName. -}
|
||||
| newvariety == oldvariety && not (hasExt oldvariety) &&
|
||||
keyHash oldkey /= fromKey keyName oldkey =
|
||||
keyHash oldkey /= S.fromShort (fromKey keyName oldkey) =
|
||||
Just $ alterKey oldkey $ \d -> d
|
||||
{ keyName = keyHash oldkey
|
||||
{ keyName = S.toShort (keyHash oldkey)
|
||||
}
|
||||
| otherwise = Nothing
|
||||
where
|
||||
|
|
|
@ -16,6 +16,7 @@ import Types.Key
|
|||
import Types.KeySource
|
||||
|
||||
import qualified Data.ByteString as S
|
||||
import qualified Data.ByteString.Short as S (ShortByteString, toShort)
|
||||
import qualified Data.ByteString.Lazy as L
|
||||
import qualified System.FilePath.ByteString as P
|
||||
import Data.Char
|
||||
|
@ -25,13 +26,13 @@ import Data.Word
|
|||
- If it's not too long, the full string is used as the keyName.
|
||||
- Otherwise, it's truncated, and its md5 is prepended to ensure a unique
|
||||
- key. -}
|
||||
genKeyName :: String -> S.ByteString
|
||||
genKeyName :: String -> S.ShortByteString
|
||||
genKeyName s
|
||||
-- Avoid making keys longer than the length of a SHA256 checksum.
|
||||
| bytelen > sha256len = encodeBS $
|
||||
| bytelen > sha256len = S.toShort $ encodeBS $
|
||||
truncateFilePath (sha256len - md5len - 1) s' ++ "-" ++
|
||||
show (md5 bl)
|
||||
| otherwise = encodeBS s'
|
||||
| otherwise = S.toShort $ encodeBS s'
|
||||
where
|
||||
s' = preSanitizeKeyName s
|
||||
bl = encodeBL s
|
||||
|
@ -47,7 +48,7 @@ addE source sethasext k = do
|
|||
maxlen <- annexMaxExtensionLength <$> Annex.getGitConfig
|
||||
let ext = selectExtension maxlen (keyFilename source)
|
||||
return $ alterKey k $ \d -> d
|
||||
{ keyName = keyName d <> ext
|
||||
{ keyName = keyName d <> S.toShort ext
|
||||
, keyVariety = sethasext (keyVariety d)
|
||||
}
|
||||
|
||||
|
|
|
@ -17,6 +17,7 @@ import Utility.Metered
|
|||
|
||||
import qualified Data.ByteString.Char8 as S8
|
||||
import qualified Utility.RawFilePath as R
|
||||
import qualified Data.ByteString.Short as S (toShort, fromShort)
|
||||
|
||||
backends :: [Backend]
|
||||
backends = [backend]
|
||||
|
@ -53,12 +54,13 @@ keyValue source _ = do
|
|||
{- Old WORM keys could contain spaces and carriage returns,
|
||||
- and can be upgraded to remove them. -}
|
||||
needsUpgrade :: Key -> Bool
|
||||
needsUpgrade key = any (`S8.elem` fromKey keyName key) [' ', '\r']
|
||||
needsUpgrade key =
|
||||
any (`S8.elem` S.fromShort (fromKey keyName key)) [' ', '\r']
|
||||
|
||||
removeProblemChars :: Key -> Backend -> AssociatedFile -> Annex (Maybe Key)
|
||||
removeProblemChars oldkey newbackend _
|
||||
| migratable = return $ Just $ alterKey oldkey $ \d -> d
|
||||
{ keyName = encodeBS $ reSanitizeKeyName $ decodeBS $ keyName d }
|
||||
{ keyName = S.toShort $ encodeBS $ reSanitizeKeyName $ decodeBS $ S.fromShort $ keyName d }
|
||||
| otherwise = return Nothing
|
||||
where
|
||||
migratable = oldvariety == newvariety
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue