Added backends for the BLAKE2 family of hashes.

There are a lot of different variants and sizes, I suppose we might as well
export all the common ones.

Bump dep to cryptonite to 0.16, earlier versions lacked BLAKE2 support.
Even android has 0.16 or newer.

On Debian, Blake2bp_512 is buggy, so I have omitted it for now.
http://bugs.debian.org/892855

This commit was sponsored by andrea rota.

Backend/Hash.hs

@@ -1,6 +1,6 @@
 {- git-annex hashing backends
  -
- - Copyright 2011-2017 Joey Hess <id@joeyh.name>
+ - Copyright 2011-2018 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -28,6 +28,9 @@ data Hash
 	| SHA2Hash HashSize
 	| SHA3Hash HashSize
 	| SkeinHash HashSize
+	| Blake2bHash HashSize
+	| Blake2sHash HashSize
+	| Blake2spHash HashSize
 
 {- Order is slightly significant; want SHA256 first, and more general
  - sizes earlier. -}
@@ -36,6 +39,9 @@ hashes = concat
 	[ map (SHA2Hash . HashSize) [256, 512, 224, 384]
 	, map (SHA3Hash . HashSize) [256, 512, 224, 384]
 	, map (SkeinHash . HashSize) [256, 512]
+	, map (Blake2bHash . HashSize) [256, 512, 160, 224, 384]
+	, map (Blake2sHash . HashSize) [256, 160, 224]
+	, map (Blake2spHash . HashSize) [256, 224]
 	, [SHA1Hash]
 	, [MD5Hash]
 	]
@@ -66,6 +72,9 @@ hashKeyVariety SHA1Hash = SHA1Key
 hashKeyVariety (SHA2Hash size) = SHA2Key size
 hashKeyVariety (SHA3Hash size) = SHA3Key size
 hashKeyVariety (SkeinHash size) = SKEINKey size
+hashKeyVariety (Blake2bHash size) = Blake2bKey size
+hashKeyVariety (Blake2sHash size) = Blake2sKey size
+hashKeyVariety (Blake2spHash size) = Blake2spKey size
 
 {- A key is a hash of its contents. -}
 keyValue :: Hash -> KeySource -> Annex (Maybe Key)
@@ -168,6 +177,9 @@ hashFile hash file filesize = go hash
 	go (SHA2Hash hashsize) = usehasher hashsize
 	go (SHA3Hash hashsize) = use (sha3Hasher hashsize)
 	go (SkeinHash hashsize) = use (skeinHasher hashsize)
+	go (Blake2bHash hashsize) = use (blake2bHasher hashsize)
+	go (Blake2sHash hashsize) = use (blake2sHasher hashsize)
+	go (Blake2spHash hashsize) = use (blake2spHasher hashsize)
 	
 	use hasher = liftIO $ do
 		h <- hasher <$> L.readFile file
@@ -219,6 +231,28 @@ skeinHasher (HashSize hashsize)
 	| hashsize == 512 = show . skein512
 	| otherwise = error $ "unsupported SKEIN size " ++ show hashsize
 
+blake2bHasher :: HashSize -> (L.ByteString -> String)
+blake2bHasher (HashSize hashsize)
+	| hashsize == 256 = show . blake2b_256
+	| hashsize == 512 = show . blake2b_512
+	| hashsize == 160 = show . blake2b_160
+	| hashsize == 224 = show . blake2b_224
+	| hashsize == 384 = show . blake2b_384
+	| otherwise = error $ "unsupported BLAKE2B size " ++ show hashsize
+
+blake2sHasher :: HashSize -> (L.ByteString -> String)
+blake2sHasher (HashSize hashsize)
+	| hashsize == 256 = show . blake2s_256
+	| hashsize == 160 = show . blake2s_160
+	| hashsize == 224 = show . blake2s_224
+	| otherwise = error $ "unsupported BLAKE2S size " ++ show hashsize
+
+blake2spHasher :: HashSize -> (L.ByteString -> String)
+blake2spHasher (HashSize hashsize)
+	| hashsize == 256 = show . blake2sp_256
+	| hashsize == 224 = show . blake2sp_224
+	| otherwise = error $ "unsupported BLAKE2SP size " ++ show hashsize
+
 md5Hasher :: L.ByteString -> String
 md5Hasher = show . md5
 

CHANGELOG

@@ -15,6 +15,7 @@ git-annex (6.20180228) UNRELEASED; urgency=medium
     This is done in some edge cases where there's a likelyhood than an
     object was downloaded incorrectly.
   * Support exporttree=yes for rsync special remotes.
+  * Added backends for the BLAKE2 family of hashes.
   * Dial back optimisation when building on arm, which prevents
     ghc and llc from running out of memory when optimising some files.
   * Improve SHA*E extension extraction code to not treat parts of the

Types/Key.hs

@@ -31,6 +31,9 @@ data KeyVariety
 	= SHA2Key HashSize HasExt
 	| SHA3Key HashSize HasExt
 	| SKEINKey HashSize HasExt
+	| Blake2bKey HashSize HasExt
+	| Blake2sKey HashSize HasExt
+	| Blake2spKey HashSize HasExt
 	| SHA1Key HasExt
 	| MD5Key HasExt
 	| WORMKey
@@ -52,6 +55,9 @@ hasExt :: KeyVariety -> Bool
 hasExt (SHA2Key _ (HasExt b)) = b
 hasExt (SHA3Key _ (HasExt b)) = b
 hasExt (SKEINKey _ (HasExt b)) = b
+hasExt (Blake2bKey _ (HasExt b)) = b
+hasExt (Blake2sKey _ (HasExt b)) = b
+hasExt (Blake2spKey _ (HasExt b)) = b
 hasExt (SHA1Key (HasExt b)) = b
 hasExt (MD5Key (HasExt b)) = b
 hasExt WORMKey = False
@@ -62,6 +68,9 @@ sameExceptExt :: KeyVariety -> KeyVariety -> Bool
 sameExceptExt (SHA2Key sz1 _) (SHA2Key sz2 _) = sz1 == sz2
 sameExceptExt (SHA3Key sz1 _) (SHA3Key sz2 _) = sz1 == sz2
 sameExceptExt (SKEINKey sz1 _) (SKEINKey sz2 _) = sz1 == sz2
+sameExceptExt (Blake2bKey sz1 _) (Blake2bKey sz2 _) = sz1 == sz2
+sameExceptExt (Blake2sKey sz1 _) (Blake2sKey sz2 _) = sz1 == sz2
+sameExceptExt (Blake2spKey sz1 _) (Blake2spKey sz2 _) = sz1 == sz2
 sameExceptExt (SHA1Key _) (SHA1Key _) = True
 sameExceptExt (MD5Key _) (MD5Key _) = True
 sameExceptExt _ _ = False
@@ -72,6 +81,9 @@ cryptographicallySecure :: KeyVariety -> Bool
 cryptographicallySecure (SHA2Key _ _) = True
 cryptographicallySecure (SHA3Key _ _) = True
 cryptographicallySecure (SKEINKey _ _) = True
+cryptographicallySecure (Blake2bKey _ _) = True
+cryptographicallySecure (Blake2sKey _ _) = True
+cryptographicallySecure (Blake2spKey _ _) = True
 cryptographicallySecure _ = False
 
 formatKeyVariety :: KeyVariety -> String
@@ -79,6 +91,9 @@ formatKeyVariety v = case v of
 	SHA2Key sz e -> adde e (addsz sz "SHA")
 	SHA3Key sz e -> adde e (addsz sz "SHA3_")
 	SKEINKey sz e -> adde e (addsz sz "SKEIN")
+	Blake2bKey sz e -> adde e (addsz sz "BLAKE2B")
+	Blake2sKey sz e -> adde e (addsz sz "BLAKE2S")
+	Blake2spKey sz e -> adde e (addsz sz "BLAKE2SP")
 	SHA1Key e -> adde e "SHA1"
 	MD5Key e -> adde e "MD5"
 	WORMKey -> "WORM"
@@ -110,6 +125,26 @@ parseKeyVariety "SKEIN512"  = SKEINKey (HashSize 512) (HasExt False)
 parseKeyVariety "SKEIN512E"    = SKEINKey (HashSize 512) (HasExt True)
 parseKeyVariety "SKEIN256"     = SKEINKey (HashSize 256) (HasExt False)
 parseKeyVariety "SKEIN256E"    = SKEINKey (HashSize 256) (HasExt True)
+parseKeyVariety "BLAKE2B160"   = Blake2bKey (HashSize 160) (HasExt False)
+parseKeyVariety "BLAKE2B160E"  = Blake2bKey (HashSize 160) (HasExt True)
+parseKeyVariety "BLAKE2B224"   = Blake2bKey (HashSize 224) (HasExt False)
+parseKeyVariety "BLAKE2B224E"  = Blake2bKey (HashSize 224) (HasExt True)
+parseKeyVariety "BLAKE2B256"   = Blake2bKey (HashSize 256) (HasExt False)
+parseKeyVariety "BLAKE2B256E"  = Blake2bKey (HashSize 256) (HasExt True)
+parseKeyVariety "BLAKE2B384"   = Blake2bKey (HashSize 384) (HasExt False)
+parseKeyVariety "BLAKE2B384E"  = Blake2bKey (HashSize 384) (HasExt True)
+parseKeyVariety "BLAKE2B512"   = Blake2bKey (HashSize 512) (HasExt False)
+parseKeyVariety "BLAKE2B512E"  = Blake2bKey (HashSize 512) (HasExt True)
+parseKeyVariety "BLAKE2S160"   = Blake2sKey (HashSize 160) (HasExt False)
+parseKeyVariety "BLAKE2S160E"  = Blake2sKey (HashSize 160) (HasExt True)
+parseKeyVariety "BLAKE2S224"   = Blake2sKey (HashSize 224) (HasExt False)
+parseKeyVariety "BLAKE2S224E"  = Blake2sKey (HashSize 224) (HasExt True)
+parseKeyVariety "BLAKE2S256"   = Blake2sKey (HashSize 256) (HasExt False)
+parseKeyVariety "BLAKE2S256E"  = Blake2sKey (HashSize 256) (HasExt True)
+parseKeyVariety "BLAKE2SP224"  = Blake2spKey (HashSize 224) (HasExt False)
+parseKeyVariety "BLAKE2SP224E" = Blake2spKey (HashSize 224) (HasExt True)
+parseKeyVariety "BLAKE2SP256"  = Blake2spKey (HashSize 256) (HasExt False)
+parseKeyVariety "BLAKE2SP256E" = Blake2spKey (HashSize 256) (HasExt True)
 parseKeyVariety "SHA1"        = SHA1Key (HasExt False)
 parseKeyVariety "SHA1E"       = SHA1Key (HasExt True)
 parseKeyVariety "MD5"         = MD5Key (HasExt False)

Utility/Hash.hs

@@ -12,6 +12,16 @@ module Utility.Hash (
 	sha3_512,
 	skein256,
 	skein512,
+	blake2s_160,
+	blake2s_224,
+	blake2s_256,
+	blake2sp_224,
+	blake2sp_256,
+	blake2b_160,
+	blake2b_224,
+	blake2b_256,
+	blake2b_384,
+	blake2b_512,
 	md5,
 	prop_hashes_stable,
 	Mac(..),
@@ -59,6 +69,40 @@ skein256 = hashlazy
 skein512 :: L.ByteString -> Digest Skein512_512
 skein512 = hashlazy
 
+blake2s_160 :: L.ByteString -> Digest Blake2s_160
+blake2s_160 = hashlazy
+
+blake2s_224 :: L.ByteString -> Digest Blake2s_224
+blake2s_224 = hashlazy
+
+blake2s_256 :: L.ByteString -> Digest Blake2s_256
+blake2s_256 = hashlazy
+
+blake2sp_224 :: L.ByteString -> Digest Blake2sp_224
+blake2sp_224 = hashlazy
+
+blake2sp_256 :: L.ByteString -> Digest Blake2sp_256
+blake2sp_256 = hashlazy
+
+blake2b_160 :: L.ByteString -> Digest Blake2b_160
+blake2b_160 = hashlazy
+
+blake2b_224 :: L.ByteString -> Digest Blake2b_224
+blake2b_224 = hashlazy
+
+blake2b_256 :: L.ByteString -> Digest Blake2b_256
+blake2b_256 = hashlazy
+
+blake2b_384 :: L.ByteString -> Digest Blake2b_384
+blake2b_384 = hashlazy
+
+blake2b_512 :: L.ByteString -> Digest Blake2b_512
+blake2b_512 = hashlazy
+
+-- Disabled because it's buggy with some versions of cryptonite.
+--blake2bp_512 :: L.ByteString -> Digest Blake2bp_512
+--blake2bp_512 = hashlazy
+
 md5 ::  L.ByteString -> Digest MD5
 md5 = hashlazy
 
@@ -76,6 +120,17 @@ prop_hashes_stable = all (\(hasher, result) -> hasher foo == result)
 	, (show . sha3_256, "76d3bc41c9f588f7fcd0d5bf4718f8f84b1c41b20882703100b9eb9413807c01")
 	, (show . sha3_384, "665551928d13b7d84ee02734502b018d896a0fb87eed5adb4c87ba91bbd6489410e11b0fbcc06ed7d0ebad559e5d3bb5")
 	, (show . sha3_512, "4bca2b137edc580fe50a88983ef860ebaca36c857b1f492839d6d7392452a63c82cbebc68e3b70a2a1480b4bb5d437a7cba6ecf9d89f9ff3ccd14cd6146ea7e7")
+	, (show . blake2s_160, "52fb63154f958a5c56864597273ea759e52c6f00")
+	, (show . blake2s_224, "9466668503ac415d87b8e1dfd7f348ab273ac1d5e4f774fced5fdb55")
+	, (show . blake2s_256, "08d6cad88075de8f192db097573d0e829411cd91eb6ec65e8fc16c017edfdb74")
+	, (show . blake2sp_224, "8492d356fbac99f046f55e114301f7596649cb590e5b083d1a19dcdb")
+	, (show . blake2sp_256, "050dc5786037ea72cb9ed9d0324afcab03c97ec02e8c47368fc5dfb4cf49d8c9")
+	, (show . blake2b_160, "983ceba2afea8694cc933336b27b907f90c53a88")
+	, (show . blake2b_224, "853986b3fe231d795261b4fb530e1a9188db41e460ec4ca59aafef78")
+	, (show . blake2b_256, "b8fe9f7f6255a6fa08f668ab632a8d081ad87983c77cd274e48ce450f0b349fd")
+	, (show . blake2b_384, "e629ee880953d32c8877e479e3b4cb0a4c9d5805e2b34c675b5a5863c4ad7d64bb2a9b8257fac9d82d289b3d39eb9cc2")
+	, (show . blake2b_512, "ca002330e69d3e6b84a46a56a6533fd79d51d97a3bb7cad6c2ff43b354185d6dc1e723fb3db4ae0737e120378424c714bb982d9dc5bbd7a0ab318240ddd18f8d")
+	--, (show . blake2bp_512, "")
 	, (show . md5, "acbd18db4cc2f85cedef654fccc4a4d8")
 	]
   where

doc/backends.mdwn

@@ -22,6 +22,18 @@ These are the recommended backends to use.
 * `SKEIN512`, `SKEIN512E`, `SKEIN256`, `SKEIN256E`
   -- [Skein hash](http://en.wikipedia.org/wiki/Skein_hash),
   a well-regarded SHA3 hash competition finalist.
+* `BLAKE2B160`, `BLAKE2B224`, `BLAKE2B256`, `BLAKE2B384`, `BLAKE2B512`
+  `BLAKE2B160E`, `BLAKE2B224E`, `BLAKE2B256E`, `BLAKE2B384E`, `BLAKE2B512E`
+  -- Fast [Blake2 hash](https://blake2.net/) variants optimised for 64 bit
+  platforms.
+* `BLAKE2S160`, `BLAKE2S224`, `BLAKE2S256`
+  `BLAKE2S160E`, `BLAKE2S224E`, `BLAKE2S256E`
+  -- Fast [Blake2 hash](https://blake2.net/) variants optimised for 32 bit
+  platforms.
+* `BLAKE2SP224`, `BLAKE2SP256`
+  `BLAKE2SP224E`, `BLAKE2SP256E`
+  -- Fast [Blake2 hash](https://blake2.net/) variants optimised for 
+  8-way CPUs.
 
 The backends below do not guarantee cryptographically that the
 content of an annexed file remains unchanged.

doc/todo/support_disabling_verification_of_transfer_over_p2p_protocol/comment_2_22831e802e6e4b6a95e1901699c11d09._comment

@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2018-03-13T19:10:38Z"
+ content="""
+The hash speed is not really significant; re-reading the content of the
+file to hash it is the expensive part. At some point git-annex may gain the
+ability to do this on the fly as it's being downloaded and then the
+verification overhead would be negligible.
+
+Thanks for mentioning BLAKE2.. I've added support for it!
+"""]]

git-annex.cabal

@@ -360,7 +360,7 @@ Executable git-annex
    stm-chans,
    securemem,
    crypto-api,
-   cryptonite,
+   cryptonite (>= 0.16),
    memory,
    split,
    QuickCheck (>= 2.1),