enforce retrievalSecurityPolicy

Leveraged the existing verification code by making it also check the
retrievalSecurityPolicy.

Also, prevented getViaTmp from running the download action at all when the
retrievalSecurityPolicy is going to prevent verifying and so storing it.

Added annex.security.allow-unverified-downloads. A per-remote version
would be nice to have too, but would need more plumbing, so KISS.
(Bill the Cat reference not too over the top I hope. The point is to
make this something the user reads the documentation for before using.)

A few calls to verifyKeyContent and getViaTmp, that don't
involve downloads from remotes, have RetrievalAllKeysSecure hard-coded.
It was also hard-coded for P2P.Annex and Command.RecvKey,
to match the values of the corresponding remotes.

A few things use retrieveKeyFile/retrieveKeyFileCheap without going
through getViaTmp.
* Command.Fsck when downloading content from a remote to verify it.
  That content does not get into the annex, so this is ok.
* Command.AddUrl when using a remote to download an url; this is new
  content being added, so this is ok.

This commit was sponsored by Fernando Jimenez on Patreon.

Command/Get.hs

@@ -109,7 +109,7 @@ getKey' key afile = dispatch
 		| Remote.hasKeyCheap r =
 			either (const False) id <$> Remote.hasKey r key
 		| otherwise = return True
-	docopy r witness = getViaTmp (RemoteVerify r) key $ \dest ->
+	docopy r witness = getViaTmp (Remote.retrievalSecurityPolicy r) (RemoteVerify r) key $ \dest ->
 		download (Remote.uuid r) key afile stdRetry
 			(\p -> do
 				showAction $ "from " ++ Remote.name r

Command/Move.hs

@@ -207,7 +207,7 @@ fromPerform src removewhen key afile = do
   where
 	go = notifyTransfer Download afile $ 
 		download (Remote.uuid src) key afile stdRetry $ \p ->
-			getViaTmp (RemoteVerify src) key $ \t ->
+			getViaTmp (Remote.retrievalSecurityPolicy src) (RemoteVerify src) key $ \t ->
 				Remote.retrieveKeyFile src key afile t p
 	dispatch _ _ False = stop -- failed
 	dispatch RemoveNever _ True = next $ return True -- copy complete

Command/Multicast.hs

@@ -213,7 +213,7 @@ storeReceived f = do
 			warning $ "Received a file " ++ f ++ " that is not a git-annex key. Deleting this file."
 			liftIO $ nukeFile f
 		Just k -> void $
-			getViaTmpFromDisk AlwaysVerify k $ \dest -> unVerified $
+			getViaTmpFromDisk RetrievalVerifiableKeysSecure AlwaysVerify k $ \dest -> unVerified $
 				liftIO $ catchBoolIO $ do
 					rename f dest
 					return True

Command/ReKey.hs

@@ -83,7 +83,7 @@ linkKey file oldkey newkey = ifM (isJust <$> isAnnexLink file)
 	 - This avoids hard linking to content linked to an
 	 - unlocked file, which would leave the new key unlocked
 	 - and vulnerable to corruption. -}
-	( getViaTmpFromDisk DefaultVerify newkey $ \tmp -> unVerified $ do
+	( getViaTmpFromDisk RetrievalAllKeysSecure DefaultVerify newkey $ \tmp -> unVerified $ do
 		oldobj <- calcRepo (gitAnnexLocation oldkey)
 		linkOrCopy' (return True) newkey oldobj tmp Nothing
 	, do

Command/RecvKey.hs

@@ -13,6 +13,7 @@ import Annex.Action
 import Annex
 import Utility.Rsync
 import Types.Transfer
+import Types.Remote (RetrievalSecurityPolicy(..))
 import Command.SendKey (fieldTransfer)
 import qualified CmdLine.GitAnnexShell.Fields as Fields
 
@@ -31,7 +32,9 @@ start key = fieldTransfer Download key $ \_p -> do
 	fromunlocked <- (isJust <$> Fields.getField Fields.unlocked)
 		<||> (isJust <$> Fields.getField Fields.direct)
 	let verify = if fromunlocked then AlwaysVerify else DefaultVerify
-	ifM (getViaTmp verify key go)
+	-- This matches the retrievalSecurityPolicy of Remote.Git
+	let rsp = RetrievalAllKeysSecure
+	ifM (getViaTmp rsp verify key go)
 		( do
 			-- forcibly quit after receiving one key,
 			-- and shutdown cleanly

Command/Reinject.hs

@@ -45,7 +45,7 @@ startSrcDest (src:dest:[])
 		showStart "reinject" dest
 		next $ ifAnnexed dest go stop
   where
-	go key = ifM (verifyKeyContent DefaultVerify UnVerified key src)
+	go key = ifM (verifyKeyContent RetrievalAllKeysSecure DefaultVerify UnVerified key src)
 		( perform src key
 		, error "failed"
 		)

Command/SetKey.hs

@@ -33,7 +33,7 @@ perform file key = do
 	-- the file might be on a different filesystem, so moveFile is used
 	-- rather than simply calling moveAnnex; disk space is also
 	-- checked this way.
-	ok <- getViaTmp DefaultVerify key $ \dest -> unVerified $
+	ok <- getViaTmp RetrievalAllKeysSecure DefaultVerify key $ \dest -> unVerified $
 		if dest /= file
 			then liftIO $ catchBoolIO $ do
 				moveFile file dest

Command/TestRemote.hs

@@ -179,7 +179,7 @@ test st r k =
 		Just b -> case Backend.verifyKeyContent b of
 			Nothing -> return True
 			Just verifier -> verifier k (key2file k)
-	get = getViaTmp (RemoteVerify r) k $ \dest ->
+	get = getViaTmp (Remote.retrievalSecurityPolicy r) (RemoteVerify r) k $ \dest ->
 		Remote.retrieveKeyFile r k (AssociatedFile Nothing)
 			dest nullMeterUpdate
 	store = Remote.storeKey r k (AssociatedFile Nothing) nullMeterUpdate
@@ -220,7 +220,7 @@ testExportTree st (Just _) ea k1 k2 =
 	retrieveexport k = withTmpFile "exported" $ \tmp h -> do
 		liftIO $ hClose h
 		ifM (Remote.retrieveExport ea k testexportlocation tmp nullMeterUpdate)
-			( verifyKeyContent AlwaysVerify UnVerified k tmp
+			( verifyKeyContent RetrievalAllKeysSecure AlwaysVerify UnVerified k tmp
 			, return False
 			)
 	checkpresentexport k = Remote.checkPresentExport ea k testexportlocation
@@ -238,10 +238,10 @@ testUnavailable st r k =
 	, check (`notElem` [Right True, Right False]) "checkPresent" $
 		Remote.checkPresent r k
 	, check (== Right False) "retrieveKeyFile" $
-		getViaTmp (RemoteVerify r) k $ \dest ->
+		getViaTmp (Remote.retrievalSecurityPolicy r) (RemoteVerify r) k $ \dest ->
 			Remote.retrieveKeyFile r k (AssociatedFile Nothing) dest nullMeterUpdate
 	, check (== Right False) "retrieveKeyFileCheap" $
-		getViaTmp (RemoteVerify r) k $ \dest -> unVerified $
+		getViaTmp (Remote.retrievalSecurityPolicy r) (RemoteVerify r) k $ \dest -> unVerified $
 			Remote.retrieveKeyFileCheap r k (AssociatedFile Nothing) dest
 	]
   where

Command/TransferKey.hs

@@ -60,7 +60,7 @@ toPerform key file remote = go Upload file $
 fromPerform :: Key -> AssociatedFile -> Remote -> CommandPerform
 fromPerform key file remote = go Upload file $
 	download (uuid remote) key file stdRetry $ \p ->
-		getViaTmp (RemoteVerify remote) key $ 
+		getViaTmp (retrievalSecurityPolicy remote) (RemoteVerify remote) key $ 
 			\t -> Remote.retrieveKeyFile remote key file t p
 
 go :: Direction -> AssociatedFile -> (NotifyWitness -> Annex Bool) -> CommandPerform

Command/TransferKeys.hs

@@ -42,7 +42,7 @@ start = do
 				return ok
 		| otherwise = notifyTransfer direction file $
 			download (Remote.uuid remote) key file stdRetry $ \p ->
-				getViaTmp (RemoteVerify remote) key $ \t -> do
+				getViaTmp (Remote.retrievalSecurityPolicy remote) (RemoteVerify remote) key $ \t -> do
 					r <- Remote.retrieveKeyFile remote key file t p
 					-- Make sure we get the current
 					-- associated files data for the key,