git-annex/doc/bugs/blake3_hash_support.mdwn

This is a patch that seems to work for my personal use.
BLAKE3 does support variable lengths, but my code does not implement support for anything other than 256-bit (32-byte) digests.
I'm not familiar enough with the codebase to be sure whether adding variable length support later is a backwards compatibility hazard or not.

[[!format patch """
From a661bf0ba954b86909fa2f6b58727294068ac082 Mon Sep 17 00:00:00 2001
From: edef <edef@edef.eu>
Date: Fri, 2 Dec 2022 12:16:44 +0000
Subject: [PATCH v3 1/2] support BLAKE3

This uses the blake3 package from Hackage, since cryptonite does not
have BLAKE3 support yet.
---
 Backend/Hash.hs | 29 +++++++++++++++++++++++++++++
 Types/Key.hs    |  6 ++++++
 git-annex.cabal |  1 +
 stack.yaml      |  1 +
 4 files changed, 37 insertions(+)

diff --git a/Backend/Hash.hs b/Backend/Hash.hs
index 550d8fc6c..809a82599 100644
--- a/Backend/Hash.hs
+++ b/Backend/Hash.hs
@@ -27,8 +27,11 @@ import qualified Data.ByteString as S
 import qualified Data.ByteString.Short as S (toShort, fromShort)
 import qualified Data.ByteString.Char8 as S8
 import qualified Data.ByteString.Lazy as L
+import Data.IORef
+import Control.Arrow
 import Control.DeepSeq
 import Control.Exception (evaluate)
+import qualified BLAKE3
 
 data Hash
 	= MD5Hash
@@ -40,6 +43,7 @@ data Hash
 	| Blake2bpHash HashSize
 	| Blake2sHash HashSize
 	| Blake2spHash HashSize
+	| Blake3Hash
 
 cryptographicallySecure :: Hash -> Bool
 cryptographicallySecure (SHA2Hash _) = True
@@ -49,6 +53,7 @@ cryptographicallySecure (Blake2bHash _) = True
 cryptographicallySecure (Blake2bpHash _) = True
 cryptographicallySecure (Blake2sHash _) = True
 cryptographicallySecure (Blake2spHash _) = True
+cryptographicallySecure Blake3Hash = True
 cryptographicallySecure SHA1Hash = False
 cryptographicallySecure MD5Hash = False
 
@@ -63,6 +68,7 @@ hashes = concat
 	, map (Blake2bpHash . HashSize) [512]
 	, map (Blake2sHash . HashSize) [256, 160, 224]
 	, map (Blake2spHash . HashSize) [256, 224]
+	, [Blake3Hash]
 	, [SHA1Hash]
 	, [MD5Hash]
 	]
@@ -99,6 +105,7 @@ hashKeyVariety (Blake2bHash size) he = Blake2bKey size he
 hashKeyVariety (Blake2bpHash size) he = Blake2bpKey size he
 hashKeyVariety (Blake2sHash size) he = Blake2sKey size he
 hashKeyVariety (Blake2spHash size) he = Blake2spKey size he
+hashKeyVariety Blake3Hash he = Blake3Key he
 
 {- A key is a hash of its contents. -}
 keyValue :: Hash -> KeySource -> MeterUpdate -> Annex Key
@@ -219,6 +226,7 @@ hasher (Blake2bHash hashsize) = blake2bHasher hashsize
 hasher (Blake2bpHash hashsize) = blake2bpHasher hashsize
 hasher (Blake2sHash hashsize) = blake2sHasher hashsize
 hasher (Blake2spHash hashsize) = blake2spHasher hashsize
+hasher Blake3Hash = blake3Hasher
 
 mkHasher :: HashAlgorithm h => (L.ByteString -> Digest h) -> Context h -> Hasher
 mkHasher h c = (show . h, mkIncrementalVerifier c descChecksum . sameCheckSum)
@@ -272,6 +280,27 @@ blake2spHasher (HashSize hashsize)
 	| hashsize == 224 = mkHasher blake2sp_224 blake2sp_224_context
 	| otherwise = error $ "unsupported BLAKE2SP size " ++ show hashsize
 
+blake3Hasher :: Hasher
+blake3Hasher = (hash, incremental) where
+	finalize :: BLAKE3.Hasher -> BLAKE3.Digest BLAKE3.DEFAULT_DIGEST_LEN
+	finalize = BLAKE3.finalize
+
+	hash :: L.ByteString -> String
+	hash = show . finalize . L.foldlChunks ((. pure) . BLAKE3.update) BLAKE3.hasher
+
+	incremental :: Key -> IO IncrementalVerifier
+	incremental k = do
+		v <- newIORef (Just (BLAKE3.hasher, 0))
+		return $ IncrementalVerifier
+			{ updateIncrementalVerifier = \b ->
+				modifyIORef' v . fmap $ flip BLAKE3.update [b] *** (fromIntegral (S.length b) +)
+			, finalizeIncrementalVerifier =
+				fmap (sameCheckSum k . show . finalize . fst) <$> readIORef v
+			, unableIncrementalVerifier = writeIORef v Nothing
+			, positionIncrementalVerifier = fmap snd <$> readIORef v
+			, descIncrementalVerifier = descChecksum
+			}
+
 sha1Hasher :: Hasher
 sha1Hasher = mkHasher sha1 sha1_context
 
diff --git a/Types/Key.hs b/Types/Key.hs
index 271723982..43f64a74d 100644
--- a/Types/Key.hs
+++ b/Types/Key.hs
@@ -214,6 +214,7 @@ data KeyVariety
 	| Blake2bpKey HashSize HasExt
 	| Blake2sKey HashSize HasExt
 	| Blake2spKey HashSize HasExt
+	| Blake3Key HasExt
 	| SHA1Key HasExt
 	| MD5Key HasExt
 	| WORMKey
@@ -247,6 +248,7 @@ hasExt (Blake2bKey _ (HasExt b)) = b
 hasExt (Blake2bpKey _ (HasExt b)) = b
 hasExt (Blake2sKey _ (HasExt b)) = b
 hasExt (Blake2spKey _ (HasExt b)) = b
+hasExt (Blake3Key (HasExt b)) = b
 hasExt (SHA1Key (HasExt b)) = b
 hasExt (MD5Key (HasExt b)) = b
 hasExt WORMKey = False
@@ -262,6 +264,7 @@ sameExceptExt (Blake2bKey sz1 _) (Blake2bKey sz2 _) = sz1 == sz2
 sameExceptExt (Blake2bpKey sz1 _) (Blake2bpKey sz2 _) = sz1 == sz2
 sameExceptExt (Blake2sKey sz1 _) (Blake2sKey sz2 _) = sz1 == sz2
 sameExceptExt (Blake2spKey sz1 _) (Blake2spKey sz2 _) = sz1 == sz2
+sameExceptExt (Blake3Key _) (Blake3Key _) = True
 sameExceptExt (SHA1Key _) (SHA1Key _) = True
 sameExceptExt (MD5Key _) (MD5Key _) = True
 sameExceptExt _ _ = False
@@ -275,6 +278,7 @@ formatKeyVariety v = case v of
 	Blake2bpKey sz e -> adde e (addsz sz "BLAKE2BP")
 	Blake2sKey sz e -> adde e (addsz sz "BLAKE2S")
 	Blake2spKey sz e -> adde e (addsz sz "BLAKE2SP")
+	Blake3Key e -> adde e "BLAKE3_256"
 	SHA1Key e -> adde e "SHA1"
 	MD5Key e -> adde e "MD5"
 	WORMKey -> "WORM"
@@ -337,6 +341,8 @@ parseKeyVariety "BLAKE2SP224"  = Blake2spKey (HashSize 224) (HasExt False)
 parseKeyVariety "BLAKE2SP224E" = Blake2spKey (HashSize 224) (HasExt True)
 parseKeyVariety "BLAKE2SP256"  = Blake2spKey (HashSize 256) (HasExt False)
 parseKeyVariety "BLAKE2SP256E" = Blake2spKey (HashSize 256) (HasExt True)
+parseKeyVariety "BLAKE3_256"   = Blake3Key (HasExt False)
+parseKeyVariety "BLAKE3_256E"  = Blake3Key (HasExt True)
 parseKeyVariety "SHA1"         = SHA1Key (HasExt False)
 parseKeyVariety "SHA1E"        = SHA1Key (HasExt True)
 parseKeyVariety "MD5"          = MD5Key (HasExt False)
diff --git a/git-annex.cabal b/git-annex.cabal
index eb9b68cbc..30614e156 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -362,6 +362,7 @@ Executable git-annex
    securemem,
    crypto-api,
    cryptonite (>= 0.23),
+   blake3,
    memory,
    deepseq,
    split,
diff --git a/stack.yaml b/stack.yaml
index 7dbfb657a..936ee841b 100644
--- a/stack.yaml
+++ b/stack.yaml
@@ -25,3 +25,4 @@ extra-deps:
 - base64-bytestring-1.0.0.3
 - bencode-0.6.1.1
 - http-client-0.7.9
+- blake3-0.2@sha256:d1146b9a51ccfbb0532780778b6d016a614e3d44c05d8c1923dde9a8be869045,2448
"""]]

[[!format patch """
From 9385299c5d9d7862ec2768ef57cab1fb7f59695d Mon Sep 17 00:00:00 2001
From: edef <edef@edef.eu>
Date: Fri, 16 Dec 2022 14:37:53 +0000
Subject: [PATCH v3 2/2] make BLAKE3 support optional

This sticks it behind a build flag, since not all distros ship the
blake3 package.
---
 Backend/Hash.hs | 19 +++++++++++++++++--
 Types/Key.hs    | 11 +++++++++++
 git-annex.cabal |  9 ++++++++-
 3 files changed, 36 insertions(+), 3 deletions(-)

diff --git a/Backend/Hash.hs b/Backend/Hash.hs
index 809a82599..4e9a75725 100644
--- a/Backend/Hash.hs
+++ b/Backend/Hash.hs
@@ -5,6 +5,7 @@
  - Licensed under the GNU AGPL version 3 or higher.
  -}
 
+{-# LANGUAGE CPP #-}
 {-# LANGUAGE OverloadedStrings #-}
 
 module Backend.Hash (
@@ -27,11 +28,13 @@ import qualified Data.ByteString as S
 import qualified Data.ByteString.Short as S (toShort, fromShort)
 import qualified Data.ByteString.Char8 as S8
 import qualified Data.ByteString.Lazy as L
-import Data.IORef
-import Control.Arrow
 import Control.DeepSeq
 import Control.Exception (evaluate)
+#ifdef WITH_BLAKE3
+import Data.IORef
+import Control.Arrow
 import qualified BLAKE3
+#endif
 
 data Hash
 	= MD5Hash
@@ -43,7 +46,9 @@ data Hash
 	| Blake2bpHash HashSize
 	| Blake2sHash HashSize
 	| Blake2spHash HashSize
+#ifdef WITH_BLAKE3
 	| Blake3Hash
+#endif
 
 cryptographicallySecure :: Hash -> Bool
 cryptographicallySecure (SHA2Hash _) = True
@@ -53,7 +58,9 @@ cryptographicallySecure (Blake2bHash _) = True
 cryptographicallySecure (Blake2bpHash _) = True
 cryptographicallySecure (Blake2sHash _) = True
 cryptographicallySecure (Blake2spHash _) = True
+#ifdef WITH_BLAKE3
 cryptographicallySecure Blake3Hash = True
+#endif
 cryptographicallySecure SHA1Hash = False
 cryptographicallySecure MD5Hash = False
 
@@ -68,7 +75,9 @@ hashes = concat
 	, map (Blake2bpHash . HashSize) [512]
 	, map (Blake2sHash . HashSize) [256, 160, 224]
 	, map (Blake2spHash . HashSize) [256, 224]
+#ifdef WITH_BLAKE3
 	, [Blake3Hash]
+#endif
 	, [SHA1Hash]
 	, [MD5Hash]
 	]
@@ -105,7 +114,9 @@ hashKeyVariety (Blake2bHash size) he = Blake2bKey size he
 hashKeyVariety (Blake2bpHash size) he = Blake2bpKey size he
 hashKeyVariety (Blake2sHash size) he = Blake2sKey size he
 hashKeyVariety (Blake2spHash size) he = Blake2spKey size he
+#ifdef WITH_BLAKE3
 hashKeyVariety Blake3Hash he = Blake3Key he
+#endif
 
 {- A key is a hash of its contents. -}
 keyValue :: Hash -> KeySource -> MeterUpdate -> Annex Key
@@ -226,7 +237,9 @@ hasher (Blake2bHash hashsize) = blake2bHasher hashsize
 hasher (Blake2bpHash hashsize) = blake2bpHasher hashsize
 hasher (Blake2sHash hashsize) = blake2sHasher hashsize
 hasher (Blake2spHash hashsize) = blake2spHasher hashsize
+#ifdef WITH_BLAKE3
 hasher Blake3Hash = blake3Hasher
+#endif
 
 mkHasher :: HashAlgorithm h => (L.ByteString -> Digest h) -> Context h -> Hasher
 mkHasher h c = (show . h, mkIncrementalVerifier c descChecksum . sameCheckSum)
@@ -280,6 +293,7 @@ blake2spHasher (HashSize hashsize)
 	| hashsize == 224 = mkHasher blake2sp_224 blake2sp_224_context
 	| otherwise = error $ "unsupported BLAKE2SP size " ++ show hashsize
 
+#ifdef WITH_BLAKE3
 blake3Hasher :: Hasher
 blake3Hasher = (hash, incremental) where
 	finalize :: BLAKE3.Hasher -> BLAKE3.Digest BLAKE3.DEFAULT_DIGEST_LEN
@@ -300,6 +314,7 @@ blake3Hasher = (hash, incremental) where
 			, positionIncrementalVerifier = fmap snd <$> readIORef v
 			, descIncrementalVerifier = descChecksum
 			}
+#endif
 
 sha1Hasher :: Hasher
 sha1Hasher = mkHasher sha1 sha1_context
diff --git a/Types/Key.hs b/Types/Key.hs
index 43f64a74d..806bdc034 100644
--- a/Types/Key.hs
+++ b/Types/Key.hs
@@ -5,6 +5,7 @@
  - Licensed under the GNU AGPL version 3 or higher.
  -}
 
+{-# LANGUAGE CPP #-}
 {-# LANGUAGE OverloadedStrings, DeriveGeneric #-}
 
 module Types.Key (
@@ -214,7 +215,9 @@ data KeyVariety
 	| Blake2bpKey HashSize HasExt
 	| Blake2sKey HashSize HasExt
 	| Blake2spKey HashSize HasExt
+#ifdef WITH_BLAKE3
 	| Blake3Key HasExt
+#endif
 	| SHA1Key HasExt
 	| MD5Key HasExt
 	| WORMKey
@@ -248,7 +251,9 @@ hasExt (Blake2bKey _ (HasExt b)) = b
 hasExt (Blake2bpKey _ (HasExt b)) = b
 hasExt (Blake2sKey _ (HasExt b)) = b
 hasExt (Blake2spKey _ (HasExt b)) = b
+#ifdef WITH_BLAKE3
 hasExt (Blake3Key (HasExt b)) = b
+#endif
 hasExt (SHA1Key (HasExt b)) = b
 hasExt (MD5Key (HasExt b)) = b
 hasExt WORMKey = False
@@ -264,7 +269,9 @@ sameExceptExt (Blake2bKey sz1 _) (Blake2bKey sz2 _) = sz1 == sz2
 sameExceptExt (Blake2bpKey sz1 _) (Blake2bpKey sz2 _) = sz1 == sz2
 sameExceptExt (Blake2sKey sz1 _) (Blake2sKey sz2 _) = sz1 == sz2
 sameExceptExt (Blake2spKey sz1 _) (Blake2spKey sz2 _) = sz1 == sz2
+#ifdef WITH_BLAKE3
 sameExceptExt (Blake3Key _) (Blake3Key _) = True
+#endif
 sameExceptExt (SHA1Key _) (SHA1Key _) = True
 sameExceptExt (MD5Key _) (MD5Key _) = True
 sameExceptExt _ _ = False
@@ -278,7 +285,9 @@ formatKeyVariety v = case v of
 	Blake2bpKey sz e -> adde e (addsz sz "BLAKE2BP")
 	Blake2sKey sz e -> adde e (addsz sz "BLAKE2S")
 	Blake2spKey sz e -> adde e (addsz sz "BLAKE2SP")
+#ifdef WITH_BLAKE3
 	Blake3Key e -> adde e "BLAKE3_256"
+#endif
 	SHA1Key e -> adde e "SHA1"
 	MD5Key e -> adde e "MD5"
 	WORMKey -> "WORM"
@@ -341,8 +350,10 @@ parseKeyVariety "BLAKE2SP224"  = Blake2spKey (HashSize 224) (HasExt False)
 parseKeyVariety "BLAKE2SP224E" = Blake2spKey (HashSize 224) (HasExt True)
 parseKeyVariety "BLAKE2SP256"  = Blake2spKey (HashSize 256) (HasExt False)
 parseKeyVariety "BLAKE2SP256E" = Blake2spKey (HashSize 256) (HasExt True)
+#ifdef WITH_BLAKE3
 parseKeyVariety "BLAKE3_256"   = Blake3Key (HasExt False)
 parseKeyVariety "BLAKE3_256E"  = Blake3Key (HasExt True)
+#endif
 parseKeyVariety "SHA1"         = SHA1Key (HasExt False)
 parseKeyVariety "SHA1E"        = SHA1Key (HasExt True)
 parseKeyVariety "MD5"          = MD5Key (HasExt False)
diff --git a/git-annex.cabal b/git-annex.cabal
index 30614e156..0778948fb 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -288,6 +288,10 @@ Flag GitLfs
   Description: Build with git-lfs library (rather than vendored copy)
   Default: True
 
+Flag Blake3
+  Description: Enable blake3 support
+  Default: True
+
 source-repository head
   type: git
   location: git://git-annex.branchable.com/
@@ -362,7 +366,6 @@ Executable git-annex
    securemem,
    crypto-api,
    cryptonite (>= 0.23),
-   blake3,
    memory,
    deepseq,
    split,
@@ -572,6 +575,10 @@ Executable git-annex
       CPP-Options: -DWITH_DBUS -DWITH_DESKTOP_NOTIFY -DWITH_DBUS_NOTIFICATIONS
       Other-Modules: Utility.DBus
 
+  if flag(Blake3)
+    Build-Depends: blake3
+    CPP-Options: -DWITH_BLAKE3
+
   if flag(Pairing)
     Build-Depends: network-multicast, network-info
     CPP-Options: -DWITH_PAIRING
"""]]