mirrors/git-annex
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Added backends for the BLAKE2 family of hashes.

Browse source 
There are a lot of different variants and sizes, I suppose we might as well
export all the common ones.

Bump dep to cryptonite to 0.16, earlier versions lacked BLAKE2 support.
Even android has 0.16 or newer.

On Debian, Blake2bp_512 is buggy, so I have omitted it for now.
http://bugs.debian.org/892855

This commit was sponsored by andrea rota.
This commit is contained in:
Joey Hess 2018-03-13 16:15:35 -04:00
parent 4d8687062f
commit 050ada746f
No known key found for this signature in database
GPG key ID: DB12DB0FF05F8F38
7 changed files with 178 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
Backend/Hash.hs
View file

@ -1,6 +1,6 @@
{- git-annex hashing backends
-
- Copyright 2011-2017 Joey Hess <id@joeyh.name>
- Copyright 2011-2018 Joey Hess <id@joeyh.name>
-
- Licensed under the GNU GPL version 3 or higher.
-}
@ -28,6 +28,9 @@ data Hash
| SHA2Hash HashSize
| SHA3Hash HashSize
| SkeinHash HashSize
| Blake2bHash HashSize
| Blake2sHash HashSize
| Blake2spHash HashSize
{- Order is slightly significant; want SHA256 first, and more general
- sizes earlier. -}
@ -36,6 +39,9 @@ hashes = concat
[ map (SHA2Hash . HashSize) [256, 512, 224, 384]
, map (SHA3Hash . HashSize) [256, 512, 224, 384]
, map (SkeinHash . HashSize) [256, 512]
, map (Blake2bHash . HashSize) [256, 512, 160, 224, 384]
, map (Blake2sHash . HashSize) [256, 160, 224]
, map (Blake2spHash . HashSize) [256, 224]
, [SHA1Hash]
, [MD5Hash]
]
@ -66,6 +72,9 @@ hashKeyVariety SHA1Hash = SHA1Key
hashKeyVariety (SHA2Hash size) = SHA2Key size
hashKeyVariety (SHA3Hash size) = SHA3Key size
hashKeyVariety (SkeinHash size) = SKEINKey size
hashKeyVariety (Blake2bHash size) = Blake2bKey size
hashKeyVariety (Blake2sHash size) = Blake2sKey size
hashKeyVariety (Blake2spHash size) = Blake2spKey size
{- A key is a hash of its contents. -}
keyValue :: Hash -> KeySource -> Annex (Maybe Key)
@ -168,6 +177,9 @@ hashFile hash file filesize = go hash
go (SHA2Hash hashsize) = usehasher hashsize
go (SHA3Hash hashsize) = use (sha3Hasher hashsize)
go (SkeinHash hashsize) = use (skeinHasher hashsize)
go (Blake2bHash hashsize) = use (blake2bHasher hashsize)
go (Blake2sHash hashsize) = use (blake2sHasher hashsize)
go (Blake2spHash hashsize) = use (blake2spHasher hashsize)
use hasher = liftIO $ do
h <- hasher <$> L.readFile file
@ -219,6 +231,28 @@ skeinHasher (HashSize hashsize)
| hashsize == 512 = show . skein512
| otherwise = error $ "unsupported SKEIN size " ++ show hashsize
blake2bHasher :: HashSize -> (L.ByteString -> String)
blake2bHasher (HashSize hashsize)
| hashsize == 256 = show . blake2b_256
| hashsize == 512 = show . blake2b_512
| hashsize == 160 = show . blake2b_160
| hashsize == 224 = show . blake2b_224
| hashsize == 384 = show . blake2b_384
| otherwise = error $ "unsupported BLAKE2B size " ++ show hashsize
blake2sHasher :: HashSize -> (L.ByteString -> String)
blake2sHasher (HashSize hashsize)
| hashsize == 256 = show . blake2s_256
| hashsize == 160 = show . blake2s_160
| hashsize == 224 = show . blake2s_224
| otherwise = error $ "unsupported BLAKE2S size " ++ show hashsize
blake2spHasher :: HashSize -> (L.ByteString -> String)
blake2spHasher (HashSize hashsize)
| hashsize == 256 = show . blake2sp_256
| hashsize == 224 = show . blake2sp_224
| otherwise = error $ "unsupported BLAKE2SP size " ++ show hashsize
md5Hasher :: L.ByteString -> String
md5Hasher = show . md5

1
CHANGELOG
View file

@ -15,6 +15,7 @@ git-annex (6.20180228) UNRELEASED; urgency=medium
This is done in some edge cases where there's a likelyhood than an
object was downloaded incorrectly.
* Support exporttree=yes for rsync special remotes.
* Added backends for the BLAKE2 family of hashes.
* Dial back optimisation when building on arm, which prevents
ghc and llc from running out of memory when optimising some files.
* Improve SHA*E extension extraction code to not treat parts of the

89
Types/Key.hs
View file

@ -31,6 +31,9 @@ data KeyVariety
= SHA2Key HashSize HasExt
| SHA3Key HashSize HasExt
| SKEINKey HashSize HasExt
| Blake2bKey HashSize HasExt
| Blake2sKey HashSize HasExt
| Blake2spKey HashSize HasExt
| SHA1Key HasExt
| MD5Key HasExt
| WORMKey
@ -52,6 +55,9 @@ hasExt :: KeyVariety -> Bool
hasExt (SHA2Key _ (HasExt b)) = b
hasExt (SHA3Key _ (HasExt b)) = b
hasExt (SKEINKey _ (HasExt b)) = b
hasExt (Blake2bKey _ (HasExt b)) = b
hasExt (Blake2sKey _ (HasExt b)) = b
hasExt (Blake2spKey _ (HasExt b)) = b
hasExt (SHA1Key (HasExt b)) = b
hasExt (MD5Key (HasExt b)) = b
hasExt WORMKey = False
@ -62,6 +68,9 @@ sameExceptExt :: KeyVariety -> KeyVariety -> Bool
sameExceptExt (SHA2Key sz1 _) (SHA2Key sz2 _) = sz1 == sz2
sameExceptExt (SHA3Key sz1 _) (SHA3Key sz2 _) = sz1 == sz2
sameExceptExt (SKEINKey sz1 _) (SKEINKey sz2 _) = sz1 == sz2
sameExceptExt (Blake2bKey sz1 _) (Blake2bKey sz2 _) = sz1 == sz2
sameExceptExt (Blake2sKey sz1 _) (Blake2sKey sz2 _) = sz1 == sz2
sameExceptExt (Blake2spKey sz1 _) (Blake2spKey sz2 _) = sz1 == sz2
sameExceptExt (SHA1Key _) (SHA1Key _) = True
sameExceptExt (MD5Key _) (MD5Key _) = True
sameExceptExt _ _ = False
@ -72,6 +81,9 @@ cryptographicallySecure :: KeyVariety -> Bool
cryptographicallySecure (SHA2Key _ _) = True
cryptographicallySecure (SHA3Key _ _) = True
cryptographicallySecure (SKEINKey _ _) = True
cryptographicallySecure (Blake2bKey _ _) = True
cryptographicallySecure (Blake2sKey _ _) = True
cryptographicallySecure (Blake2spKey _ _) = True
cryptographicallySecure _ = False
formatKeyVariety :: KeyVariety -> String
@ -79,6 +91,9 @@ formatKeyVariety v = case v of
SHA2Key sz e -> adde e (addsz sz "SHA")
SHA3Key sz e -> adde e (addsz sz "SHA3_")
SKEINKey sz e -> adde e (addsz sz "SKEIN")
Blake2bKey sz e -> adde e (addsz sz "BLAKE2B")
Blake2sKey sz e -> adde e (addsz sz "BLAKE2S")
Blake2spKey sz e -> adde e (addsz sz "BLAKE2SP")
SHA1Key e -> adde e "SHA1"
MD5Key e -> adde e "MD5"
WORMKey -> "WORM"
@ -90,30 +105,50 @@ formatKeyVariety v = case v of
addsz (HashSize n) s = s ++ show n
parseKeyVariety :: String -> KeyVariety
parseKeyVariety "SHA256" = SHA2Key (HashSize 256) (HasExt False)
parseKeyVariety "SHA256E" = SHA2Key (HashSize 256) (HasExt True)
parseKeyVariety "SHA512" = SHA2Key (HashSize 512) (HasExt False)
parseKeyVariety "SHA512E" = SHA2Key (HashSize 512) (HasExt True)
parseKeyVariety "SHA224" = SHA2Key (HashSize 224) (HasExt False)
parseKeyVariety "SHA224E" = SHA2Key (HashSize 224) (HasExt True)
parseKeyVariety "SHA384" = SHA2Key (HashSize 384) (HasExt False)
parseKeyVariety "SHA384E" = SHA2Key (HashSize 384) (HasExt True)
parseKeyVariety "SHA3_512" = SHA3Key (HashSize 512) (HasExt False)
parseKeyVariety "SHA3_512E" = SHA3Key (HashSize 512) (HasExt True)
parseKeyVariety "SHA3_384" = SHA3Key (HashSize 384) (HasExt False)
parseKeyVariety "SHA3_384E" = SHA3Key (HashSize 384) (HasExt True)
parseKeyVariety "SHA3_256" = SHA3Key (HashSize 256) (HasExt False)
parseKeyVariety "SHA3_256E" = SHA3Key (HashSize 256) (HasExt True)
parseKeyVariety "SHA3_224" = SHA3Key (HashSize 224) (HasExt False)
parseKeyVariety "SHA3_224E" = SHA3Key (HashSize 224) (HasExt True)
parseKeyVariety "SKEIN512" = SKEINKey (HashSize 512) (HasExt False)
parseKeyVariety "SKEIN512E" = SKEINKey (HashSize 512) (HasExt True)
parseKeyVariety "SKEIN256" = SKEINKey (HashSize 256) (HasExt False)
parseKeyVariety "SKEIN256E" = SKEINKey (HashSize 256) (HasExt True)
parseKeyVariety "SHA1" = SHA1Key (HasExt False)
parseKeyVariety "SHA1E" = SHA1Key (HasExt True)
parseKeyVariety "MD5" = MD5Key (HasExt False)
parseKeyVariety "MD5E" = MD5Key (HasExt True)
parseKeyVariety "WORM" = WORMKey
parseKeyVariety "URL" = URLKey
parseKeyVariety s = OtherKey s
parseKeyVariety "SHA256" = SHA2Key (HashSize 256) (HasExt False)
parseKeyVariety "SHA256E" = SHA2Key (HashSize 256) (HasExt True)
parseKeyVariety "SHA512" = SHA2Key (HashSize 512) (HasExt False)
parseKeyVariety "SHA512E" = SHA2Key (HashSize 512) (HasExt True)
parseKeyVariety "SHA224" = SHA2Key (HashSize 224) (HasExt False)
parseKeyVariety "SHA224E" = SHA2Key (HashSize 224) (HasExt True)
parseKeyVariety "SHA384" = SHA2Key (HashSize 384) (HasExt False)
parseKeyVariety "SHA384E" = SHA2Key (HashSize 384) (HasExt True)
parseKeyVariety "SHA3_512" = SHA3Key (HashSize 512) (HasExt False)
parseKeyVariety "SHA3_512E" = SHA3Key (HashSize 512) (HasExt True)
parseKeyVariety "SHA3_384" = SHA3Key (HashSize 384) (HasExt False)
parseKeyVariety "SHA3_384E" = SHA3Key (HashSize 384) (HasExt True)
parseKeyVariety "SHA3_256" = SHA3Key (HashSize 256) (HasExt False)
parseKeyVariety "SHA3_256E" = SHA3Key (HashSize 256) (HasExt True)
parseKeyVariety "SHA3_224" = SHA3Key (HashSize 224) (HasExt False)
parseKeyVariety "SHA3_224E" = SHA3Key (HashSize 224) (HasExt True)
parseKeyVariety "SKEIN512" = SKEINKey (HashSize 512) (HasExt False)
parseKeyVariety "SKEIN512E" = SKEINKey (HashSize 512) (HasExt True)
parseKeyVariety "SKEIN256" = SKEINKey (HashSize 256) (HasExt False)
parseKeyVariety "SKEIN256E" = SKEINKey (HashSize 256) (HasExt True)
parseKeyVariety "BLAKE2B160" = Blake2bKey (HashSize 160) (HasExt False)
parseKeyVariety "BLAKE2B160E" = Blake2bKey (HashSize 160) (HasExt True)
parseKeyVariety "BLAKE2B224" = Blake2bKey (HashSize 224) (HasExt False)
parseKeyVariety "BLAKE2B224E" = Blake2bKey (HashSize 224) (HasExt True)
parseKeyVariety "BLAKE2B256" = Blake2bKey (HashSize 256) (HasExt False)
parseKeyVariety "BLAKE2B256E" = Blake2bKey (HashSize 256) (HasExt True)
parseKeyVariety "BLAKE2B384" = Blake2bKey (HashSize 384) (HasExt False)
parseKeyVariety "BLAKE2B384E" = Blake2bKey (HashSize 384) (HasExt True)
parseKeyVariety "BLAKE2B512" = Blake2bKey (HashSize 512) (HasExt False)
parseKeyVariety "BLAKE2B512E" = Blake2bKey (HashSize 512) (HasExt True)
parseKeyVariety "BLAKE2S160" = Blake2sKey (HashSize 160) (HasExt False)
parseKeyVariety "BLAKE2S160E" = Blake2sKey (HashSize 160) (HasExt True)
parseKeyVariety "BLAKE2S224" = Blake2sKey (HashSize 224) (HasExt False)
parseKeyVariety "BLAKE2S224E" = Blake2sKey (HashSize 224) (HasExt True)
parseKeyVariety "BLAKE2S256" = Blake2sKey (HashSize 256) (HasExt False)
parseKeyVariety "BLAKE2S256E" = Blake2sKey (HashSize 256) (HasExt True)
parseKeyVariety "BLAKE2SP224" = Blake2spKey (HashSize 224) (HasExt False)
parseKeyVariety "BLAKE2SP224E" = Blake2spKey (HashSize 224) (HasExt True)
parseKeyVariety "BLAKE2SP256" = Blake2spKey (HashSize 256) (HasExt False)
parseKeyVariety "BLAKE2SP256E" = Blake2spKey (HashSize 256) (HasExt True)
parseKeyVariety "SHA1" = SHA1Key (HasExt False)
parseKeyVariety "SHA1E" = SHA1Key (HasExt True)
parseKeyVariety "MD5" = MD5Key (HasExt False)
parseKeyVariety "MD5E" = MD5Key (HasExt True)
parseKeyVariety "WORM" = WORMKey
parseKeyVariety "URL" = URLKey
parseKeyVariety s = OtherKey s

55
Utility/Hash.hs
View file

@ -12,6 +12,16 @@ module Utility.Hash (
sha3_512,
skein256,
skein512,
blake2s_160,
blake2s_224,
blake2s_256,
blake2sp_224,
blake2sp_256,
blake2b_160,
blake2b_224,
blake2b_256,
blake2b_384,
blake2b_512,
md5,
prop_hashes_stable,
Mac(..),
@ -59,6 +69,40 @@ skein256 = hashlazy
skein512 :: L.ByteString -> Digest Skein512_512
skein512 = hashlazy
blake2s_160 :: L.ByteString -> Digest Blake2s_160
blake2s_160 = hashlazy
blake2s_224 :: L.ByteString -> Digest Blake2s_224
blake2s_224 = hashlazy
blake2s_256 :: L.ByteString -> Digest Blake2s_256
blake2s_256 = hashlazy
blake2sp_224 :: L.ByteString -> Digest Blake2sp_224
blake2sp_224 = hashlazy
blake2sp_256 :: L.ByteString -> Digest Blake2sp_256
blake2sp_256 = hashlazy
blake2b_160 :: L.ByteString -> Digest Blake2b_160
blake2b_160 = hashlazy
blake2b_224 :: L.ByteString -> Digest Blake2b_224
blake2b_224 = hashlazy
blake2b_256 :: L.ByteString -> Digest Blake2b_256
blake2b_256 = hashlazy
blake2b_384 :: L.ByteString -> Digest Blake2b_384
blake2b_384 = hashlazy
blake2b_512 :: L.ByteString -> Digest Blake2b_512
blake2b_512 = hashlazy
-- Disabled because it's buggy with some versions of cryptonite.
--blake2bp_512 :: L.ByteString -> Digest Blake2bp_512
--blake2bp_512 = hashlazy
md5 :: L.ByteString -> Digest MD5
md5 = hashlazy
@ -76,6 +120,17 @@ prop_hashes_stable = all (\(hasher, result) -> hasher foo == result)
, (show . sha3_256, "76d3bc41c9f588f7fcd0d5bf4718f8f84b1c41b20882703100b9eb9413807c01")
, (show . sha3_384, "665551928d13b7d84ee02734502b018d896a0fb87eed5adb4c87ba91bbd6489410e11b0fbcc06ed7d0ebad559e5d3bb5")
, (show . sha3_512, "4bca2b137edc580fe50a88983ef860ebaca36c857b1f492839d6d7392452a63c82cbebc68e3b70a2a1480b4bb5d437a7cba6ecf9d89f9ff3ccd14cd6146ea7e7")
, (show . blake2s_160, "52fb63154f958a5c56864597273ea759e52c6f00")
, (show . blake2s_224, "9466668503ac415d87b8e1dfd7f348ab273ac1d5e4f774fced5fdb55")
, (show . blake2s_256, "08d6cad88075de8f192db097573d0e829411cd91eb6ec65e8fc16c017edfdb74")
, (show . blake2sp_224, "8492d356fbac99f046f55e114301f7596649cb590e5b083d1a19dcdb")
, (show . blake2sp_256, "050dc5786037ea72cb9ed9d0324afcab03c97ec02e8c47368fc5dfb4cf49d8c9")
, (show . blake2b_160, "983ceba2afea8694cc933336b27b907f90c53a88")
, (show . blake2b_224, "853986b3fe231d795261b4fb530e1a9188db41e460ec4ca59aafef78")
, (show . blake2b_256, "b8fe9f7f6255a6fa08f668ab632a8d081ad87983c77cd274e48ce450f0b349fd")
, (show . blake2b_384, "e629ee880953d32c8877e479e3b4cb0a4c9d5805e2b34c675b5a5863c4ad7d64bb2a9b8257fac9d82d289b3d39eb9cc2")
, (show . blake2b_512, "ca002330e69d3e6b84a46a56a6533fd79d51d97a3bb7cad6c2ff43b354185d6dc1e723fb3db4ae0737e120378424c714bb982d9dc5bbd7a0ab318240ddd18f8d")
--, (show . blake2bp_512, "")
, (show . md5, "acbd18db4cc2f85cedef654fccc4a4d8")
]
where

12
doc/backends.mdwn
View file

@ -22,6 +22,18 @@ These are the recommended backends to use.
* `SKEIN512`, `SKEIN512E`, `SKEIN256`, `SKEIN256E`
-- [Skein hash](http://en.wikipedia.org/wiki/Skein_hash),
a well-regarded SHA3 hash competition finalist.
* `BLAKE2B160`, `BLAKE2B224`, `BLAKE2B256`, `BLAKE2B384`, `BLAKE2B512`
`BLAKE2B160E`, `BLAKE2B224E`, `BLAKE2B256E`, `BLAKE2B384E`, `BLAKE2B512E`
-- Fast [Blake2 hash](https://blake2.net/) variants optimised for 64 bit
platforms.
* `BLAKE2S160`, `BLAKE2S224`, `BLAKE2S256`
`BLAKE2S160E`, `BLAKE2S224E`, `BLAKE2S256E`
-- Fast [Blake2 hash](https://blake2.net/) variants optimised for 32 bit
platforms.
* `BLAKE2SP224`, `BLAKE2SP256`
`BLAKE2SP224E`, `BLAKE2SP256E`
-- Fast [Blake2 hash](https://blake2.net/) variants optimised for
8-way CPUs.
The backends below do not guarantee cryptographically that the
content of an annexed file remains unchanged.

12
doc/todo/support_disabling_verification_of_transfer_over_p2p_protocol/comment_2_22831e802e6e4b6a95e1901699c11d09._comment Normal file
View file

@ -0,0 +1,12 @@
[[!comment format=mdwn
username="joey"
subject="""comment 2"""
date="2018-03-13T19:10:38Z"
content="""
The hash speed is not really significant; re-reading the content of the
file to hash it is the expensive part. At some point git-annex may gain the
ability to do this on the fly as it's being downloaded and then the
verification overhead would be negligible.
Thanks for mentioning BLAKE2.. I've added support for it!
"""]]

2
git-annex.cabal
View file

@ -360,7 +360,7 @@ Executable git-annex
stm-chans,
securemem,
crypto-api,
cryptonite,
cryptonite (>= 0.16),
memory,
split,
QuickCheck (>= 2.1),