fsck: Warn when required content is not present in the repository that requires it.

This commit was sponsored by Jack Hill on Patreon.

CHANGELOG

@@ -6,6 +6,8 @@ git-annex (6.20180113) UNRELEASED; urgency=medium
   * datalad < 0.9.1 had a problem in its special remote protocol handling
     which is broken by EXTENSIONS. Make the debian git-annex package
     conflict with the problem version of datalad.
+  * fsck: Warn when required content is not present in the repository that
+    requires it.
 
  -- Joey Hess <id@joeyh.name>  Wed, 24 Jan 2018 20:42:55 -0400
 

Command/Fsck.hs

@@ -1,6 +1,6 @@
 {- git-annex command
  -
- - Copyright 2010-2017 Joey Hess <id@joeyh.name>
+ - Copyright 2010-2018 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -23,6 +23,7 @@ import Logs.Location
 import Logs.Trust
 import Logs.Activity
 import Logs.TimeStamp
+import Logs.PreferredContent
 import Annex.NumCopies
 import Annex.UUID
 import Annex.ReplaceFile
@@ -40,6 +41,8 @@ import Types.ActionItem
 
 import Data.Time.Clock.POSIX
 import System.Posix.Types (EpochTime)
+import qualified Data.Set as S
+import qualified Data.Map as M
 
 cmd :: Command
 cmd = withGlobalOptions (jobsOption : jsonOption : annexedMatchingOptions) $
@@ -121,6 +124,7 @@ perform key file backend numcopies = do
 		-- order matters
 		[ fixLink key file
 		, verifyLocationLog key keystatus ai
+		, verifyRequiredContent key ai
 		, verifyAssociatedFiles key keystatus file
 		, verifyWorkTree key file
 		, checkKeySize key keystatus ai
@@ -151,6 +155,7 @@ performRemote key afile backend numcopies remote =
 	dispatch (Right False) = go False Nothing
 	go present localcopy = check
 		[ verifyLocationLogRemote key ai remote present
+		, verifyRequiredContent key ai
 		, withLocalCopy localcopy $ checkKeySizeRemote key remote ai
 		, withLocalCopy localcopy $ checkBackendRemote backend key remote ai
 		, checkKeyNumCopies key afile numcopies
@@ -286,6 +291,27 @@ verifyLocationLog' key ai present u updatestatus = do
 		showNote "fixing location log"
 		updatestatus s
 
+{- Verifies that all repos that are required to contain the content do,
+ - checking against the location log. -}
+verifyRequiredContent :: Key -> ActionItem -> Annex Bool
+verifyRequiredContent key ai@(ActionItemAssociatedFile afile) = do
+	presentlocs <- S.fromList <$> loggedLocations key
+	requiredlocs <- S.fromList . M.keys <$> requiredContentMap
+	missinglocs <- filterM
+		(\u -> isRequiredContent (Just u) S.empty (Just key) afile False)
+		(S.toList $ S.difference requiredlocs presentlocs)
+	if null missinglocs
+		then return True
+		else do
+			missingrequired <- Remote.prettyPrintUUIDs "missingrequired" missinglocs
+			warning $
+				"** Required content " ++
+				actionItemDesc ai key ++
+				" is missing from these repositories:\n" ++
+				missingrequired
+			return False
+verifyRequiredContent _ _ = return True
+
 {- Verifies the associated file records. -}
 verifyAssociatedFiles :: Key -> KeyStatus -> FilePath -> Annex Bool
 verifyAssociatedFiles key keystatus file = do

doc/forum/offsite_repo/comment_1_1f3ab9a39baa16e3e307b23fd1aabeb8._comment

@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2018-02-08T17:32:00Z"
+ content="""
+What you're looking for is probably the [[required_content]] settings.
+
+However, `git annex fsck` did not perform any checks that required content
+was present. Now it does. Enjoy!
+"""]]

doc/git-annex-required.mdwn

@@ -24,6 +24,9 @@ removed. For example a file that is `wanted` can be removed with
 `git annex drop`, but if that file is `required`, it would need to be
 removed with `git annex drop --force`. 
 
+Also, `git-annex fsck` will warn about required contents that are not
+present.
+
 # NOTES
 
 The `required` command was added in git-annex 5.20150420.

doc/required_content.mdwn

@@ -16,3 +16,7 @@ by simply using `git annex drop`. On the other hand, required content
 settings are enforced; `git annex drop` will refuse to drop a file if
 doing so would violate its required content settings.
 (Although even this can be overridden using `--force`).
+
+Also, `git-annex fsck` will warn about required contents that are not
+present.
+