don't count clusters as copies
Since the cluster UUID is inserted into the location log when the location log lists a node as containing content. Also avoid trying to lock content on cluster remotes. The cluster nodes are also proxied, so that content can be locked on individual nodes, and locking content on a cluster as a whole probably won't be implemented. And made git-annex whereis use numcopies machinery for displaying its count, so it won't count cluster UUIDs redundantly to nodes. Other commands, like git-annex info that also display numcopies information already used the numcopies machinery. There is more to be done, fromNumCopies is sometimes used to get a number that is compared with a list of UUIDs. And limitCopies doesn't use numcopies machinery.
This commit is contained in:
parent
b3370a191c
commit
36c6d8da69
2 changed files with 28 additions and 10 deletions
|
@ -1,6 +1,6 @@
|
||||||
{- git-annex numcopies configuration and checking
|
{- git-annex numcopies configuration and checking
|
||||||
-
|
-
|
||||||
- Copyright 2014-2021 Joey Hess <id@joeyh.name>
|
- Copyright 2014-2024 Joey Hess <id@joeyh.name>
|
||||||
-
|
-
|
||||||
- Licensed under the GNU AGPL version 3 or higher.
|
- Licensed under the GNU AGPL version 3 or higher.
|
||||||
-}
|
-}
|
||||||
|
@ -20,6 +20,7 @@ module Annex.NumCopies (
|
||||||
defaultNumCopies,
|
defaultNumCopies,
|
||||||
numCopiesCheck,
|
numCopiesCheck,
|
||||||
numCopiesCheck',
|
numCopiesCheck',
|
||||||
|
numCopiesCount,
|
||||||
verifyEnoughCopiesToDrop,
|
verifyEnoughCopiesToDrop,
|
||||||
verifiableCopies,
|
verifiableCopies,
|
||||||
UnVerifiedCopy(..),
|
UnVerifiedCopy(..),
|
||||||
|
@ -30,6 +31,7 @@ import qualified Annex
|
||||||
import Types.NumCopies
|
import Types.NumCopies
|
||||||
import Logs.NumCopies
|
import Logs.NumCopies
|
||||||
import Logs.Trust
|
import Logs.Trust
|
||||||
|
import Types.Cluster
|
||||||
import Annex.CheckAttr
|
import Annex.CheckAttr
|
||||||
import qualified Remote
|
import qualified Remote
|
||||||
import qualified Types.Remote as Remote
|
import qualified Types.Remote as Remote
|
||||||
|
@ -198,12 +200,19 @@ numCopiesCheck file key vs = do
|
||||||
numCopiesCheck' :: RawFilePath -> (Int -> Int -> v) -> [UUID] -> Annex v
|
numCopiesCheck' :: RawFilePath -> (Int -> Int -> v) -> [UUID] -> Annex v
|
||||||
numCopiesCheck' file vs have = do
|
numCopiesCheck' file vs have = do
|
||||||
needed <- fromNumCopies . fst <$> getFileNumMinCopies file
|
needed <- fromNumCopies . fst <$> getFileNumMinCopies file
|
||||||
let nhave = length have
|
let nhave = numCopiesCount have
|
||||||
explain (ActionItemTreeFile file) $ Just $ UnquotedString $
|
explain (ActionItemTreeFile file) $ Just $ UnquotedString $
|
||||||
"has " ++ show nhave ++ " " ++ pluralCopies nhave ++
|
"has " ++ show nhave ++ " " ++ pluralCopies nhave ++
|
||||||
", and the configured annex.numcopies is " ++ show needed
|
", and the configured annex.numcopies is " ++ show needed
|
||||||
return $ nhave `vs` needed
|
return $ nhave `vs` needed
|
||||||
|
|
||||||
|
{- When a key is logged as present in a node of the cluster,
|
||||||
|
- the cluster's UUID will also be in the list, but is not a
|
||||||
|
- distinct copy.
|
||||||
|
-}
|
||||||
|
numCopiesCount :: [UUID] -> Int
|
||||||
|
numCopiesCount = length . filter (not . isClusterUUID)
|
||||||
|
|
||||||
data UnVerifiedCopy = UnVerifiedRemote Remote | UnVerifiedHere
|
data UnVerifiedCopy = UnVerifiedRemote Remote | UnVerifiedHere
|
||||||
deriving (Ord, Eq)
|
deriving (Ord, Eq)
|
||||||
|
|
||||||
|
@ -239,12 +248,17 @@ verifyEnoughCopiesToDrop nolocmsg key removallock neednum needmin skip preverifi
|
||||||
Left stillhave -> helper bad missing stillhave (c:cs) lockunsupported
|
Left stillhave -> helper bad missing stillhave (c:cs) lockunsupported
|
||||||
| otherwise = case c of
|
| otherwise = case c of
|
||||||
UnVerifiedHere -> lockContentShared key contverified
|
UnVerifiedHere -> lockContentShared key contverified
|
||||||
UnVerifiedRemote r -> checkremote r contverified $
|
UnVerifiedRemote r
|
||||||
let lockunsupported' = r : lockunsupported
|
-- Skip cluster uuids because locking is
|
||||||
in Remote.hasKey r key >>= \case
|
-- not supported with them, instead will
|
||||||
Right True -> helper bad missing (mkVerifiedCopy RecentlyVerifiedCopy r : have) cs lockunsupported'
|
-- lock individual nodes.
|
||||||
Left _ -> helper (r:bad) missing have cs lockunsupported'
|
| isClusterUUID (Remote.uuid r) -> helper bad missing have cs lockunsupported
|
||||||
Right False -> helper bad (Remote.uuid r:missing) have cs lockunsupported'
|
| otherwise -> checkremote r contverified $
|
||||||
|
let lockunsupported' = r : lockunsupported
|
||||||
|
in Remote.hasKey r key >>= \case
|
||||||
|
Right True -> helper bad missing (mkVerifiedCopy RecentlyVerifiedCopy r : have) cs lockunsupported'
|
||||||
|
Left _ -> helper (r:bad) missing have cs lockunsupported'
|
||||||
|
Right False -> helper bad (Remote.uuid r:missing) have cs lockunsupported'
|
||||||
where
|
where
|
||||||
contverified vc = helper bad missing (vc : have) cs lockunsupported
|
contverified vc = helper bad missing (vc : have) cs lockunsupported
|
||||||
|
|
||||||
|
@ -312,13 +326,16 @@ pluralCopies _ = "copies"
|
||||||
- The return lists also exclude any repositories that are untrusted,
|
- The return lists also exclude any repositories that are untrusted,
|
||||||
- since those should not be used for verification.
|
- since those should not be used for verification.
|
||||||
-
|
-
|
||||||
|
- Cluster UUIDs are also excluded since locking on a cluster is done by
|
||||||
|
- locking on individual nodes.
|
||||||
|
-
|
||||||
- The UnVerifiedCopy list is cost ordered.
|
- The UnVerifiedCopy list is cost ordered.
|
||||||
- The VerifiedCopy list contains repositories that are trusted to
|
- The VerifiedCopy list contains repositories that are trusted to
|
||||||
- contain the key.
|
- contain the key.
|
||||||
-}
|
-}
|
||||||
verifiableCopies :: Key -> [UUID] -> Annex ([UnVerifiedCopy], [VerifiedCopy])
|
verifiableCopies :: Key -> [UUID] -> Annex ([UnVerifiedCopy], [VerifiedCopy])
|
||||||
verifiableCopies key exclude = do
|
verifiableCopies key exclude = do
|
||||||
locs <- Remote.keyLocations key
|
locs <- filter (not . isClusterUUID) <$> Remote.keyLocations key
|
||||||
(remotes, trusteduuids) <- Remote.remoteLocations (Remote.IncludeIgnored False) locs
|
(remotes, trusteduuids) <- Remote.remoteLocations (Remote.IncludeIgnored False) locs
|
||||||
=<< trustGet Trusted
|
=<< trustGet Trusted
|
||||||
untrusteduuids <- trustGet UnTrusted
|
untrusteduuids <- trustGet UnTrusted
|
||||||
|
|
|
@ -15,6 +15,7 @@ import Logs.Trust
|
||||||
import Logs.Web
|
import Logs.Web
|
||||||
import Remote.Web (getWebUrls)
|
import Remote.Web (getWebUrls)
|
||||||
import Annex.UUID
|
import Annex.UUID
|
||||||
|
import Annex.NumCopies
|
||||||
import qualified Utility.Format
|
import qualified Utility.Format
|
||||||
import qualified Command.Find
|
import qualified Command.Find
|
||||||
|
|
||||||
|
@ -86,7 +87,7 @@ perform o remotemap key ai = do
|
||||||
(untrustedlocations, safelocations) <- trustPartition UnTrusted locations
|
(untrustedlocations, safelocations) <- trustPartition UnTrusted locations
|
||||||
case formatOption o of
|
case formatOption o of
|
||||||
Nothing -> do
|
Nothing -> do
|
||||||
let num = length safelocations
|
let num = numCopiesCount safelocations
|
||||||
showNote $ UnquotedString $ show num ++ " " ++ copiesplural num
|
showNote $ UnquotedString $ show num ++ " " ++ copiesplural num
|
||||||
pp <- ppwhereis "whereis" safelocations urls
|
pp <- ppwhereis "whereis" safelocations urls
|
||||||
unless (null safelocations) $
|
unless (null safelocations) $
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue