From 050ada746f5b91d6f084df27ad62ed89d5db32f4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 13 Mar 2018 16:15:35 -0400 Subject: [PATCH] Added backends for the BLAKE2 family of hashes. There are a lot of different variants and sizes, I suppose we might as well export all the common ones. Bump dep to cryptonite to 0.16, earlier versions lacked BLAKE2 support. Even android has 0.16 or newer. On Debian, Blake2bp_512 is buggy, so I have omitted it for now. http://bugs.debian.org/892855 This commit was sponsored by andrea rota. --- Backend/Hash.hs | 36 +++++++- CHANGELOG | 1 + Types/Key.hs | 89 +++++++++++++------ Utility/Hash.hs | 55 ++++++++++++ doc/backends.mdwn | 12 +++ ..._22831e802e6e4b6a95e1901699c11d09._comment | 12 +++ git-annex.cabal | 2 +- 7 files changed, 178 insertions(+), 29 deletions(-) create mode 100644 doc/todo/support_disabling_verification_of_transfer_over_p2p_protocol/comment_2_22831e802e6e4b6a95e1901699c11d09._comment diff --git a/Backend/Hash.hs b/Backend/Hash.hs index 1d54368235..4bf20a0a9f 100644 --- a/Backend/Hash.hs +++ b/Backend/Hash.hs @@ -1,6 +1,6 @@ {- git-annex hashing backends - - - Copyright 2011-2017 Joey Hess + - Copyright 2011-2018 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} @@ -28,6 +28,9 @@ data Hash | SHA2Hash HashSize | SHA3Hash HashSize | SkeinHash HashSize + | Blake2bHash HashSize + | Blake2sHash HashSize + | Blake2spHash HashSize {- Order is slightly significant; want SHA256 first, and more general - sizes earlier. -} @@ -36,6 +39,9 @@ hashes = concat [ map (SHA2Hash . HashSize) [256, 512, 224, 384] , map (SHA3Hash . HashSize) [256, 512, 224, 384] , map (SkeinHash . HashSize) [256, 512] + , map (Blake2bHash . HashSize) [256, 512, 160, 224, 384] + , map (Blake2sHash . HashSize) [256, 160, 224] + , map (Blake2spHash . HashSize) [256, 224] , [SHA1Hash] , [MD5Hash] ] @@ -66,6 +72,9 @@ hashKeyVariety SHA1Hash = SHA1Key hashKeyVariety (SHA2Hash size) = SHA2Key size hashKeyVariety (SHA3Hash size) = SHA3Key size hashKeyVariety (SkeinHash size) = SKEINKey size +hashKeyVariety (Blake2bHash size) = Blake2bKey size +hashKeyVariety (Blake2sHash size) = Blake2sKey size +hashKeyVariety (Blake2spHash size) = Blake2spKey size {- A key is a hash of its contents. -} keyValue :: Hash -> KeySource -> Annex (Maybe Key) @@ -168,6 +177,9 @@ hashFile hash file filesize = go hash go (SHA2Hash hashsize) = usehasher hashsize go (SHA3Hash hashsize) = use (sha3Hasher hashsize) go (SkeinHash hashsize) = use (skeinHasher hashsize) + go (Blake2bHash hashsize) = use (blake2bHasher hashsize) + go (Blake2sHash hashsize) = use (blake2sHasher hashsize) + go (Blake2spHash hashsize) = use (blake2spHasher hashsize) use hasher = liftIO $ do h <- hasher <$> L.readFile file @@ -219,6 +231,28 @@ skeinHasher (HashSize hashsize) | hashsize == 512 = show . skein512 | otherwise = error $ "unsupported SKEIN size " ++ show hashsize +blake2bHasher :: HashSize -> (L.ByteString -> String) +blake2bHasher (HashSize hashsize) + | hashsize == 256 = show . blake2b_256 + | hashsize == 512 = show . blake2b_512 + | hashsize == 160 = show . blake2b_160 + | hashsize == 224 = show . blake2b_224 + | hashsize == 384 = show . blake2b_384 + | otherwise = error $ "unsupported BLAKE2B size " ++ show hashsize + +blake2sHasher :: HashSize -> (L.ByteString -> String) +blake2sHasher (HashSize hashsize) + | hashsize == 256 = show . blake2s_256 + | hashsize == 160 = show . blake2s_160 + | hashsize == 224 = show . blake2s_224 + | otherwise = error $ "unsupported BLAKE2S size " ++ show hashsize + +blake2spHasher :: HashSize -> (L.ByteString -> String) +blake2spHasher (HashSize hashsize) + | hashsize == 256 = show . blake2sp_256 + | hashsize == 224 = show . blake2sp_224 + | otherwise = error $ "unsupported BLAKE2SP size " ++ show hashsize + md5Hasher :: L.ByteString -> String md5Hasher = show . md5 diff --git a/CHANGELOG b/CHANGELOG index 012bd9098f..6d007141e1 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -15,6 +15,7 @@ git-annex (6.20180228) UNRELEASED; urgency=medium This is done in some edge cases where there's a likelyhood than an object was downloaded incorrectly. * Support exporttree=yes for rsync special remotes. + * Added backends for the BLAKE2 family of hashes. * Dial back optimisation when building on arm, which prevents ghc and llc from running out of memory when optimising some files. * Improve SHA*E extension extraction code to not treat parts of the diff --git a/Types/Key.hs b/Types/Key.hs index 44ebe3ca0a..2aa7cb945f 100644 --- a/Types/Key.hs +++ b/Types/Key.hs @@ -31,6 +31,9 @@ data KeyVariety = SHA2Key HashSize HasExt | SHA3Key HashSize HasExt | SKEINKey HashSize HasExt + | Blake2bKey HashSize HasExt + | Blake2sKey HashSize HasExt + | Blake2spKey HashSize HasExt | SHA1Key HasExt | MD5Key HasExt | WORMKey @@ -52,6 +55,9 @@ hasExt :: KeyVariety -> Bool hasExt (SHA2Key _ (HasExt b)) = b hasExt (SHA3Key _ (HasExt b)) = b hasExt (SKEINKey _ (HasExt b)) = b +hasExt (Blake2bKey _ (HasExt b)) = b +hasExt (Blake2sKey _ (HasExt b)) = b +hasExt (Blake2spKey _ (HasExt b)) = b hasExt (SHA1Key (HasExt b)) = b hasExt (MD5Key (HasExt b)) = b hasExt WORMKey = False @@ -62,6 +68,9 @@ sameExceptExt :: KeyVariety -> KeyVariety -> Bool sameExceptExt (SHA2Key sz1 _) (SHA2Key sz2 _) = sz1 == sz2 sameExceptExt (SHA3Key sz1 _) (SHA3Key sz2 _) = sz1 == sz2 sameExceptExt (SKEINKey sz1 _) (SKEINKey sz2 _) = sz1 == sz2 +sameExceptExt (Blake2bKey sz1 _) (Blake2bKey sz2 _) = sz1 == sz2 +sameExceptExt (Blake2sKey sz1 _) (Blake2sKey sz2 _) = sz1 == sz2 +sameExceptExt (Blake2spKey sz1 _) (Blake2spKey sz2 _) = sz1 == sz2 sameExceptExt (SHA1Key _) (SHA1Key _) = True sameExceptExt (MD5Key _) (MD5Key _) = True sameExceptExt _ _ = False @@ -72,6 +81,9 @@ cryptographicallySecure :: KeyVariety -> Bool cryptographicallySecure (SHA2Key _ _) = True cryptographicallySecure (SHA3Key _ _) = True cryptographicallySecure (SKEINKey _ _) = True +cryptographicallySecure (Blake2bKey _ _) = True +cryptographicallySecure (Blake2sKey _ _) = True +cryptographicallySecure (Blake2spKey _ _) = True cryptographicallySecure _ = False formatKeyVariety :: KeyVariety -> String @@ -79,6 +91,9 @@ formatKeyVariety v = case v of SHA2Key sz e -> adde e (addsz sz "SHA") SHA3Key sz e -> adde e (addsz sz "SHA3_") SKEINKey sz e -> adde e (addsz sz "SKEIN") + Blake2bKey sz e -> adde e (addsz sz "BLAKE2B") + Blake2sKey sz e -> adde e (addsz sz "BLAKE2S") + Blake2spKey sz e -> adde e (addsz sz "BLAKE2SP") SHA1Key e -> adde e "SHA1" MD5Key e -> adde e "MD5" WORMKey -> "WORM" @@ -90,30 +105,50 @@ formatKeyVariety v = case v of addsz (HashSize n) s = s ++ show n parseKeyVariety :: String -> KeyVariety -parseKeyVariety "SHA256" = SHA2Key (HashSize 256) (HasExt False) -parseKeyVariety "SHA256E" = SHA2Key (HashSize 256) (HasExt True) -parseKeyVariety "SHA512" = SHA2Key (HashSize 512) (HasExt False) -parseKeyVariety "SHA512E" = SHA2Key (HashSize 512) (HasExt True) -parseKeyVariety "SHA224" = SHA2Key (HashSize 224) (HasExt False) -parseKeyVariety "SHA224E" = SHA2Key (HashSize 224) (HasExt True) -parseKeyVariety "SHA384" = SHA2Key (HashSize 384) (HasExt False) -parseKeyVariety "SHA384E" = SHA2Key (HashSize 384) (HasExt True) -parseKeyVariety "SHA3_512" = SHA3Key (HashSize 512) (HasExt False) -parseKeyVariety "SHA3_512E" = SHA3Key (HashSize 512) (HasExt True) -parseKeyVariety "SHA3_384" = SHA3Key (HashSize 384) (HasExt False) -parseKeyVariety "SHA3_384E" = SHA3Key (HashSize 384) (HasExt True) -parseKeyVariety "SHA3_256" = SHA3Key (HashSize 256) (HasExt False) -parseKeyVariety "SHA3_256E" = SHA3Key (HashSize 256) (HasExt True) -parseKeyVariety "SHA3_224" = SHA3Key (HashSize 224) (HasExt False) -parseKeyVariety "SHA3_224E" = SHA3Key (HashSize 224) (HasExt True) -parseKeyVariety "SKEIN512" = SKEINKey (HashSize 512) (HasExt False) -parseKeyVariety "SKEIN512E" = SKEINKey (HashSize 512) (HasExt True) -parseKeyVariety "SKEIN256" = SKEINKey (HashSize 256) (HasExt False) -parseKeyVariety "SKEIN256E" = SKEINKey (HashSize 256) (HasExt True) -parseKeyVariety "SHA1" = SHA1Key (HasExt False) -parseKeyVariety "SHA1E" = SHA1Key (HasExt True) -parseKeyVariety "MD5" = MD5Key (HasExt False) -parseKeyVariety "MD5E" = MD5Key (HasExt True) -parseKeyVariety "WORM" = WORMKey -parseKeyVariety "URL" = URLKey -parseKeyVariety s = OtherKey s +parseKeyVariety "SHA256" = SHA2Key (HashSize 256) (HasExt False) +parseKeyVariety "SHA256E" = SHA2Key (HashSize 256) (HasExt True) +parseKeyVariety "SHA512" = SHA2Key (HashSize 512) (HasExt False) +parseKeyVariety "SHA512E" = SHA2Key (HashSize 512) (HasExt True) +parseKeyVariety "SHA224" = SHA2Key (HashSize 224) (HasExt False) +parseKeyVariety "SHA224E" = SHA2Key (HashSize 224) (HasExt True) +parseKeyVariety "SHA384" = SHA2Key (HashSize 384) (HasExt False) +parseKeyVariety "SHA384E" = SHA2Key (HashSize 384) (HasExt True) +parseKeyVariety "SHA3_512" = SHA3Key (HashSize 512) (HasExt False) +parseKeyVariety "SHA3_512E" = SHA3Key (HashSize 512) (HasExt True) +parseKeyVariety "SHA3_384" = SHA3Key (HashSize 384) (HasExt False) +parseKeyVariety "SHA3_384E" = SHA3Key (HashSize 384) (HasExt True) +parseKeyVariety "SHA3_256" = SHA3Key (HashSize 256) (HasExt False) +parseKeyVariety "SHA3_256E" = SHA3Key (HashSize 256) (HasExt True) +parseKeyVariety "SHA3_224" = SHA3Key (HashSize 224) (HasExt False) +parseKeyVariety "SHA3_224E" = SHA3Key (HashSize 224) (HasExt True) +parseKeyVariety "SKEIN512" = SKEINKey (HashSize 512) (HasExt False) +parseKeyVariety "SKEIN512E" = SKEINKey (HashSize 512) (HasExt True) +parseKeyVariety "SKEIN256" = SKEINKey (HashSize 256) (HasExt False) +parseKeyVariety "SKEIN256E" = SKEINKey (HashSize 256) (HasExt True) +parseKeyVariety "BLAKE2B160" = Blake2bKey (HashSize 160) (HasExt False) +parseKeyVariety "BLAKE2B160E" = Blake2bKey (HashSize 160) (HasExt True) +parseKeyVariety "BLAKE2B224" = Blake2bKey (HashSize 224) (HasExt False) +parseKeyVariety "BLAKE2B224E" = Blake2bKey (HashSize 224) (HasExt True) +parseKeyVariety "BLAKE2B256" = Blake2bKey (HashSize 256) (HasExt False) +parseKeyVariety "BLAKE2B256E" = Blake2bKey (HashSize 256) (HasExt True) +parseKeyVariety "BLAKE2B384" = Blake2bKey (HashSize 384) (HasExt False) +parseKeyVariety "BLAKE2B384E" = Blake2bKey (HashSize 384) (HasExt True) +parseKeyVariety "BLAKE2B512" = Blake2bKey (HashSize 512) (HasExt False) +parseKeyVariety "BLAKE2B512E" = Blake2bKey (HashSize 512) (HasExt True) +parseKeyVariety "BLAKE2S160" = Blake2sKey (HashSize 160) (HasExt False) +parseKeyVariety "BLAKE2S160E" = Blake2sKey (HashSize 160) (HasExt True) +parseKeyVariety "BLAKE2S224" = Blake2sKey (HashSize 224) (HasExt False) +parseKeyVariety "BLAKE2S224E" = Blake2sKey (HashSize 224) (HasExt True) +parseKeyVariety "BLAKE2S256" = Blake2sKey (HashSize 256) (HasExt False) +parseKeyVariety "BLAKE2S256E" = Blake2sKey (HashSize 256) (HasExt True) +parseKeyVariety "BLAKE2SP224" = Blake2spKey (HashSize 224) (HasExt False) +parseKeyVariety "BLAKE2SP224E" = Blake2spKey (HashSize 224) (HasExt True) +parseKeyVariety "BLAKE2SP256" = Blake2spKey (HashSize 256) (HasExt False) +parseKeyVariety "BLAKE2SP256E" = Blake2spKey (HashSize 256) (HasExt True) +parseKeyVariety "SHA1" = SHA1Key (HasExt False) +parseKeyVariety "SHA1E" = SHA1Key (HasExt True) +parseKeyVariety "MD5" = MD5Key (HasExt False) +parseKeyVariety "MD5E" = MD5Key (HasExt True) +parseKeyVariety "WORM" = WORMKey +parseKeyVariety "URL" = URLKey +parseKeyVariety s = OtherKey s diff --git a/Utility/Hash.hs b/Utility/Hash.hs index 70f826b7a5..05e9a90c3d 100644 --- a/Utility/Hash.hs +++ b/Utility/Hash.hs @@ -12,6 +12,16 @@ module Utility.Hash ( sha3_512, skein256, skein512, + blake2s_160, + blake2s_224, + blake2s_256, + blake2sp_224, + blake2sp_256, + blake2b_160, + blake2b_224, + blake2b_256, + blake2b_384, + blake2b_512, md5, prop_hashes_stable, Mac(..), @@ -59,6 +69,40 @@ skein256 = hashlazy skein512 :: L.ByteString -> Digest Skein512_512 skein512 = hashlazy +blake2s_160 :: L.ByteString -> Digest Blake2s_160 +blake2s_160 = hashlazy + +blake2s_224 :: L.ByteString -> Digest Blake2s_224 +blake2s_224 = hashlazy + +blake2s_256 :: L.ByteString -> Digest Blake2s_256 +blake2s_256 = hashlazy + +blake2sp_224 :: L.ByteString -> Digest Blake2sp_224 +blake2sp_224 = hashlazy + +blake2sp_256 :: L.ByteString -> Digest Blake2sp_256 +blake2sp_256 = hashlazy + +blake2b_160 :: L.ByteString -> Digest Blake2b_160 +blake2b_160 = hashlazy + +blake2b_224 :: L.ByteString -> Digest Blake2b_224 +blake2b_224 = hashlazy + +blake2b_256 :: L.ByteString -> Digest Blake2b_256 +blake2b_256 = hashlazy + +blake2b_384 :: L.ByteString -> Digest Blake2b_384 +blake2b_384 = hashlazy + +blake2b_512 :: L.ByteString -> Digest Blake2b_512 +blake2b_512 = hashlazy + +-- Disabled because it's buggy with some versions of cryptonite. +--blake2bp_512 :: L.ByteString -> Digest Blake2bp_512 +--blake2bp_512 = hashlazy + md5 :: L.ByteString -> Digest MD5 md5 = hashlazy @@ -76,6 +120,17 @@ prop_hashes_stable = all (\(hasher, result) -> hasher foo == result) , (show . sha3_256, "76d3bc41c9f588f7fcd0d5bf4718f8f84b1c41b20882703100b9eb9413807c01") , (show . sha3_384, "665551928d13b7d84ee02734502b018d896a0fb87eed5adb4c87ba91bbd6489410e11b0fbcc06ed7d0ebad559e5d3bb5") , (show . sha3_512, "4bca2b137edc580fe50a88983ef860ebaca36c857b1f492839d6d7392452a63c82cbebc68e3b70a2a1480b4bb5d437a7cba6ecf9d89f9ff3ccd14cd6146ea7e7") + , (show . blake2s_160, "52fb63154f958a5c56864597273ea759e52c6f00") + , (show . blake2s_224, "9466668503ac415d87b8e1dfd7f348ab273ac1d5e4f774fced5fdb55") + , (show . blake2s_256, "08d6cad88075de8f192db097573d0e829411cd91eb6ec65e8fc16c017edfdb74") + , (show . blake2sp_224, "8492d356fbac99f046f55e114301f7596649cb590e5b083d1a19dcdb") + , (show . blake2sp_256, "050dc5786037ea72cb9ed9d0324afcab03c97ec02e8c47368fc5dfb4cf49d8c9") + , (show . blake2b_160, "983ceba2afea8694cc933336b27b907f90c53a88") + , (show . blake2b_224, "853986b3fe231d795261b4fb530e1a9188db41e460ec4ca59aafef78") + , (show . blake2b_256, "b8fe9f7f6255a6fa08f668ab632a8d081ad87983c77cd274e48ce450f0b349fd") + , (show . blake2b_384, "e629ee880953d32c8877e479e3b4cb0a4c9d5805e2b34c675b5a5863c4ad7d64bb2a9b8257fac9d82d289b3d39eb9cc2") + , (show . blake2b_512, "ca002330e69d3e6b84a46a56a6533fd79d51d97a3bb7cad6c2ff43b354185d6dc1e723fb3db4ae0737e120378424c714bb982d9dc5bbd7a0ab318240ddd18f8d") + --, (show . blake2bp_512, "") , (show . md5, "acbd18db4cc2f85cedef654fccc4a4d8") ] where diff --git a/doc/backends.mdwn b/doc/backends.mdwn index c5b047eccb..3944af678b 100644 --- a/doc/backends.mdwn +++ b/doc/backends.mdwn @@ -22,6 +22,18 @@ These are the recommended backends to use. * `SKEIN512`, `SKEIN512E`, `SKEIN256`, `SKEIN256E` -- [Skein hash](http://en.wikipedia.org/wiki/Skein_hash), a well-regarded SHA3 hash competition finalist. +* `BLAKE2B160`, `BLAKE2B224`, `BLAKE2B256`, `BLAKE2B384`, `BLAKE2B512` + `BLAKE2B160E`, `BLAKE2B224E`, `BLAKE2B256E`, `BLAKE2B384E`, `BLAKE2B512E` + -- Fast [Blake2 hash](https://blake2.net/) variants optimised for 64 bit + platforms. +* `BLAKE2S160`, `BLAKE2S224`, `BLAKE2S256` + `BLAKE2S160E`, `BLAKE2S224E`, `BLAKE2S256E` + -- Fast [Blake2 hash](https://blake2.net/) variants optimised for 32 bit + platforms. +* `BLAKE2SP224`, `BLAKE2SP256` + `BLAKE2SP224E`, `BLAKE2SP256E` + -- Fast [Blake2 hash](https://blake2.net/) variants optimised for + 8-way CPUs. The backends below do not guarantee cryptographically that the content of an annexed file remains unchanged. diff --git a/doc/todo/support_disabling_verification_of_transfer_over_p2p_protocol/comment_2_22831e802e6e4b6a95e1901699c11d09._comment b/doc/todo/support_disabling_verification_of_transfer_over_p2p_protocol/comment_2_22831e802e6e4b6a95e1901699c11d09._comment new file mode 100644 index 0000000000..bef2c7c7e9 --- /dev/null +++ b/doc/todo/support_disabling_verification_of_transfer_over_p2p_protocol/comment_2_22831e802e6e4b6a95e1901699c11d09._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2018-03-13T19:10:38Z" + content=""" +The hash speed is not really significant; re-reading the content of the +file to hash it is the expensive part. At some point git-annex may gain the +ability to do this on the fly as it's being downloaded and then the +verification overhead would be negligible. + +Thanks for mentioning BLAKE2.. I've added support for it! +"""]] diff --git a/git-annex.cabal b/git-annex.cabal index f577c583a2..f7f40f0b12 100644 --- a/git-annex.cabal +++ b/git-annex.cabal @@ -360,7 +360,7 @@ Executable git-annex stm-chans, securemem, crypto-api, - cryptonite, + cryptonite (>= 0.16), memory, split, QuickCheck (>= 2.1),