From 9286769d2c46bd336257dbd6cdd4a33a0331dd3a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Aug 2023 14:31:31 -0400 Subject: [PATCH] let Remote.availability return Unavilable This is groundwork for making special remotes like borg be skipped by sync when on an offline drive. Added AVAILABILITY UNAVAILABLE reponse and the UNAVAILABLERESPONSE extension to the external special remote protocol. The extension is needed because old git-annex, if it sees that response, will display a warning message. (It does continue as if the remote is globally available, which is acceptable, and the warning is only displayed at initremote due to remote.name.annex-availability caching, but still it seemed best to make this a protocol extension.) The remote.name.annex-availability git config is no longer used any more, and is documented as such. It was only used by external special remotes to cache the availability, to avoid needing to start the external process every time. Now that availability is queried as an Annex action, the external is only started by sync (and the assistant), when they actually check availability. Sponsored-by: Nicholas Golder-Manning on Patreon --- Assistant/DaemonStatus.hs | 10 +++++-- CHANGELOG | 5 ++++ Command/Sync.hs | 8 +++--- Remote/Adb.hs | 2 +- Remote/BitTorrent.hs | 2 +- Remote/Borg.hs | 3 +- Remote/Bup.hs | 3 +- Remote/Ddar.hs | 3 +- Remote/Directory.hs | 2 +- Remote/External.hs | 28 ++++++------------- Remote/External/Types.hs | 3 ++ Remote/GCrypt.hs | 2 +- Remote/Git.hs | 25 +++++++++-------- Remote/GitLFS.hs | 2 +- Remote/Glacier.hs | 2 +- Remote/Hook.hs | 2 +- Remote/HttpAlso.hs | 2 +- Remote/P2P.hs | 2 +- Remote/Rsync.hs | 3 +- Remote/S3.hs | 2 +- Remote/Tahoe.hs | 2 +- Remote/Web.hs | 2 +- Remote/WebDAV.hs | 2 +- Types/Availability.hs | 4 +-- Types/GitConfig.hs | 3 -- Types/Remote.hs | 3 +- .../external_special_remote_protocol.mdwn | 21 +++++++++++--- doc/git-annex.mdwn | 3 +- 28 files changed, 86 insertions(+), 65 deletions(-) diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs index 1dbb27fd4e..9be4558761 100644 --- a/Assistant/DaemonStatus.hs +++ b/Assistant/DaemonStatus.hs @@ -13,6 +13,7 @@ import Assistant.Common import Assistant.Alert.Utility import Utility.Tmp import Utility.NotificationBroadcaster +import Types.Availability import Types.Transfer import Logs.Transfer import Logs.Trust @@ -59,6 +60,7 @@ calcSyncRemotes = do let (exportremotes, nonexportremotes) = partition (exportTree . Remote.config) contentremotes let isimport r = importTree (Remote.config r) || Remote.thirdPartyPopulated (Remote.remotetype r) let dataremotes = filter (not . isimport) nonexportremotes + tocloud <- anyM iscloud contentremotes return $ \dstatus -> dstatus { syncRemotes = syncable @@ -66,10 +68,14 @@ calcSyncRemotes = do , syncDataRemotes = dataremotes , exportRemotes = exportremotes , downloadRemotes = contentremotes - , syncingToCloudRemote = any iscloud contentremotes + , syncingToCloudRemote = tocloud } where - iscloud r = not (Remote.readonly r) && Remote.availability r == Remote.GloballyAvailable + iscloud r + | Remote.readonly r = pure False + | otherwise = tryNonAsync (Remote.availability r) >>= return . \case + Right GloballyAvailable -> True + _ -> False {- Updates the syncRemotes list from the list of all remotes in Annex state. -} updateSyncRemotes :: Assistant () diff --git a/CHANGELOG b/CHANGELOG index 2282356058..56b6e138e9 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -8,6 +8,11 @@ git-annex (10.20230803) UNRELEASED; urgency=medium directory does not exist. An empty tree was imported, rather than the import failing. * Stop bundling curl in the OSX dmg and linux standalone image. + * sync, assist, push, pull: Skip more types of remotes when they + are not present due to eg being on a drive that is offline. + * Added AVAILABILITY UNAVAILABLE and the UNAVAILABLERESPONSE extension + to the external special remote protocol. + * The remote.name.annex-availability git config is no longer used. -- Joey Hess Mon, 07 Aug 2023 13:04:13 -0400 diff --git a/Command/Sync.hs b/Command/Sync.hs index 0d6e0c143f..44edf2142a 100644 --- a/Command/Sync.hs +++ b/Command/Sync.hs @@ -78,6 +78,7 @@ import Annex.Import import Annex.CheckIgnore import Types.FileMatcher import Types.GitConfig +import Types.Availability import qualified Database.Export as Export import Utility.Bloom import Utility.OptParse @@ -388,10 +389,9 @@ syncRemotes' ps available = listed = concat <$> mapM Remote.byNameOrGroup ps - good r - | Remote.gitSyncableRemoteType (Remote.remotetype r) = - Remote.Git.repoAvail =<< Remote.getRepo r - | otherwise = return True + good r = tryNonAsync (Remote.availability r) >>= return . \case + Right Unavailable -> False + _ -> True fastest = fromMaybe [] . headMaybe . Remote.byCost diff --git a/Remote/Adb.hs b/Remote/Adb.hs index f70fe27bdf..7d4e38696a 100644 --- a/Remote/Adb.hs +++ b/Remote/Adb.hs @@ -113,7 +113,7 @@ gen r u rc gc rs = do , gitconfig = gc , localpath = Nothing , remotetype = remote - , availability = LocallyAvailable + , availability = pure LocallyAvailable , readonly = False , appendonly = False , untrustworthy = False diff --git a/Remote/BitTorrent.hs b/Remote/BitTorrent.hs index dcc6f29004..ccc67a25d0 100644 --- a/Remote/BitTorrent.hs +++ b/Remote/BitTorrent.hs @@ -88,7 +88,7 @@ gen r _ rc gc rs = do , readonly = True , appendonly = False , untrustworthy = False - , availability = GloballyAvailable + , availability = pure GloballyAvailable , remotetype = remote , mkUnavailable = return Nothing , getInfo = return [] diff --git a/Remote/Borg.hs b/Remote/Borg.hs index d3b152c802..ba22c8cf59 100644 --- a/Remote/Borg.hs +++ b/Remote/Borg.hs @@ -112,7 +112,8 @@ gen r u rc gc rs = do , gitconfig = gc , localpath = borgRepoLocalPath borgrepo , remotetype = remote - , availability = if borgLocal borgrepo then LocallyAvailable else GloballyAvailable + , availability = pure $ + if borgLocal borgrepo then LocallyAvailable else GloballyAvailable , readonly = False , appendonly = False -- When the user sets the appendonly field, they are diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 8bcacf9b94..c754850e22 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -97,7 +97,8 @@ gen r u rc gc rs = do then Just buprepo else Nothing , remotetype = remote - , availability = if bupLocal buprepo then LocallyAvailable else GloballyAvailable + , availability = pure $ + if bupLocal buprepo then LocallyAvailable else GloballyAvailable , readonly = False , appendonly = False , untrustworthy = False diff --git a/Remote/Ddar.hs b/Remote/Ddar.hs index 3fc3fdbafb..375920a325 100644 --- a/Remote/Ddar.hs +++ b/Remote/Ddar.hs @@ -98,7 +98,8 @@ gen r u rc gc rs = do then Just $ ddarRepoLocation ddarrepo else Nothing , remotetype = remote - , availability = if ddarLocal ddarrepo then LocallyAvailable else GloballyAvailable + , availability = pure $ + if ddarLocal ddarrepo then LocallyAvailable else GloballyAvailable , readonly = False , appendonly = False , untrustworthy = False diff --git a/Remote/Directory.hs b/Remote/Directory.hs index d234ff59cb..b030bb2c46 100644 --- a/Remote/Directory.hs +++ b/Remote/Directory.hs @@ -134,7 +134,7 @@ gen r u rc gc rs = do , readonly = False , appendonly = False , untrustworthy = False - , availability = LocallyAvailable + , availability = pure LocallyAvailable , remotetype = remote , mkUnavailable = gen r u rc (gc { remoteAnnexDirectory = Just "/dev/null" }) rs diff --git a/Remote/External.hs b/Remote/External.hs index c429dc80fa..22a9a61d32 100644 --- a/Remote/External.hs +++ b/Remote/External.hs @@ -68,7 +68,7 @@ gen r u rc gc rs | externaltype == "readonly" = do c <- parsedRemoteConfig remote rc cst <- remoteCost gc c expensiveRemoteCost - let rmt = mk c cst GloballyAvailable + let rmt = mk c cst (pure GloballyAvailable) Nothing (externalInfo externaltype) Nothing @@ -87,7 +87,6 @@ gen r u rc gc rs (Git.remoteName r) (Just rs) Annex.addCleanupAction (RemoteCleanup u) $ stopExternal external cst <- getCost external r gc c - avail <- getAvailability external r gc exportsupported <- if exportTree c then checkExportSupported' external else return False @@ -107,7 +106,7 @@ gen r u rc gc rs let cheapexportsupported = if exportsupported then exportIsSupported else exportUnsupported - let rmt = mk c cst avail + let rmt = mk c cst (getAvailability external) (Just (whereisKeyM external)) (getInfoM external) (Just (claimUrlM external)) @@ -777,25 +776,16 @@ getCost external r gc pc = return c defcst = expensiveRemoteCost -{- Caches the availability in the git config to avoid needing to start up an - - external special remote every time time just to ask it what its - - availability is. - - - - Most remotes do not bother to implement a reply to this request; +{- Most remotes do not bother to implement a reply to this request; - globally available is the default. -} -getAvailability :: External -> Git.Repo -> RemoteGitConfig -> Annex Availability -getAvailability external r gc = - maybe (catchNonAsync query (const (pure defavail))) return - (remoteAnnexAvailability gc) +getAvailability :: External -> Annex Availability +getAvailability external = catchNonAsync query (const (pure defavail)) where - query = do - avail <- handleRequest external GETAVAILABILITY Nothing $ \req -> case req of - AVAILABILITY avail -> result avail - UNSUPPORTED_REQUEST -> result defavail - _ -> Nothing - setRemoteAvailability r avail - return avail + query = handleRequest external GETAVAILABILITY Nothing $ \req -> case req of + AVAILABILITY avail -> result avail + UNSUPPORTED_REQUEST -> result defavail + _ -> Nothing defavail = GloballyAvailable claimUrlM :: External -> URLString -> Annex Bool diff --git a/Remote/External/Types.hs b/Remote/External/Types.hs index ed8aa20346..1ee29ebd6c 100644 --- a/Remote/External/Types.hs +++ b/Remote/External/Types.hs @@ -110,6 +110,7 @@ supportedExtensionList :: ExtensionList supportedExtensionList = ExtensionList [ "INFO" , "GETGITREMOTENAME" + , "UNAVAILABLERESPONSE" , asyncExtension ] @@ -447,9 +448,11 @@ instance Proto.Serializable Size where instance Proto.Serializable Availability where serialize GloballyAvailable = "GLOBAL" serialize LocallyAvailable = "LOCAL" + serialize Unavailable = "UNAVAILABLE" deserialize "GLOBAL" = Just GloballyAvailable deserialize "LOCAL" = Just LocallyAvailable + deserialize "UNAVAILABLE" = Just Unavailable deserialize _ = Nothing instance Proto.Serializable [(URLString, Size, FilePath)] where diff --git a/Remote/GCrypt.hs b/Remote/GCrypt.hs index 61e8b7210e..bf8f01ed3e 100644 --- a/Remote/GCrypt.hs +++ b/Remote/GCrypt.hs @@ -158,7 +158,7 @@ gen' r u c gc rs = do , readonly = Git.repoIsHttp r , appendonly = False , untrustworthy = False - , availability = availabilityCalc r + , availability = pure (availabilityCalc r) , remotetype = remote , mkUnavailable = return Nothing , getInfo = gitRepoInfo this diff --git a/Remote/Git.hs b/Remote/Git.hs index 387fb9d4f0..d4708e40fa 100644 --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -1,6 +1,6 @@ {- Standard git remotes. - - - Copyright 2011-2021 Joey Hess + - Copyright 2011-2023 Joey Hess - - Licensed under the GNU AGPL version 3 or higher. -} @@ -11,7 +11,6 @@ module Remote.Git ( remote, configRead, - repoAvail, onLocalRepo, ) where @@ -210,7 +209,7 @@ gen r u rc gc rs , readonly = Git.repoIsHttp r , appendonly = False , untrustworthy = False - , availability = availabilityCalc r + , availability = repoAvail r , remotetype = remote , mkUnavailable = unavailable r u rc gc rs , getInfo = gitRepoInfo new @@ -233,20 +232,24 @@ unavailable r = gen r' _ -> r -- already unavailable {- Checks relatively inexpensively if a repository is available for use. -} -repoAvail :: Git.Repo -> Annex Bool +repoAvail :: Git.Repo -> Annex Availability repoAvail r - | Git.repoIsHttp r = return True + | Git.repoIsHttp r = return GloballyAvailable | Git.GCrypt.isEncrypted r = do g <- gitRepo liftIO $ do er <- Git.GCrypt.encryptedRemote g r if Git.repoIsLocal er || Git.repoIsLocalUnknown er - then catchBoolIO $ - void (Git.Config.read er) >> return True - else return True - | Git.repoIsUrl r = return True - | Git.repoIsLocalUnknown r = return False - | otherwise = liftIO $ isJust <$> catchMaybeIO (Git.Config.read r) + then checklocal er + else return GloballyAvailable + | Git.repoIsUrl r = return GloballyAvailable + | Git.repoIsLocalUnknown r = return Unavailable + | otherwise = checklocal r + where + checklocal r' = ifM (liftIO $ isJust <$> catchMaybeIO (Git.Config.read r')) + ( return LocallyAvailable + , return Unavailable + ) {- Tries to read the config for a specified remote, updates state, and - returns the updated repo. -} diff --git a/Remote/GitLFS.hs b/Remote/GitLFS.hs index fc3459f46f..31ddc6ccef 100644 --- a/Remote/GitLFS.hs +++ b/Remote/GitLFS.hs @@ -130,7 +130,7 @@ gen r u rc gc rs = do , gitconfig = gc , localpath = Nothing , remotetype = remote - , availability = GloballyAvailable + , availability = pure GloballyAvailable , readonly = False -- content cannot be removed from a git-lfs repo , appendonly = True diff --git a/Remote/Glacier.hs b/Remote/Glacier.hs index e97ad12da4..18bd65f21b 100644 --- a/Remote/Glacier.hs +++ b/Remote/Glacier.hs @@ -103,7 +103,7 @@ gen r u rc gc rs = do , readonly = False , appendonly = False , untrustworthy = False - , availability = GloballyAvailable + , availability = pure GloballyAvailable , remotetype = remote , mkUnavailable = return Nothing , getInfo = includeCredsInfo c (AWS.creds u) $ diff --git a/Remote/Hook.hs b/Remote/Hook.hs index 49d72a8a6c..a7dd3e6593 100644 --- a/Remote/Hook.hs +++ b/Remote/Hook.hs @@ -82,7 +82,7 @@ gen r u rc gc rs = do , readonly = False , appendonly = False , untrustworthy = False - , availability = GloballyAvailable + , availability = pure GloballyAvailable , remotetype = remote , mkUnavailable = gen r u rc (gc { remoteAnnexHookType = Just "!dne!" }) diff --git a/Remote/HttpAlso.hs b/Remote/HttpAlso.hs index ff5a638ced..06003b7b54 100644 --- a/Remote/HttpAlso.hs +++ b/Remote/HttpAlso.hs @@ -95,7 +95,7 @@ gen r u rc gc rs = do , readonly = True , appendonly = False , untrustworthy = False - , availability = GloballyAvailable + , availability = pure GloballyAvailable , remotetype = remote , mkUnavailable = return Nothing , getInfo = return [] diff --git a/Remote/P2P.hs b/Remote/P2P.hs index a5fd926a36..4b47e2f489 100644 --- a/Remote/P2P.hs +++ b/Remote/P2P.hs @@ -77,7 +77,7 @@ chainGen addr r u rc gc rs = do , readonly = False , appendonly = False , untrustworthy = False - , availability = GloballyAvailable + , availability = pure GloballyAvailable , remotetype = remote , mkUnavailable = return Nothing , getInfo = gitRepoInfo this diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs index c256fe6784..b0fceeab24 100644 --- a/Remote/Rsync.hs +++ b/Remote/Rsync.hs @@ -120,7 +120,8 @@ gen r u rc gc rs = do , readonly = False , appendonly = False , untrustworthy = False - , availability = if islocal then LocallyAvailable else GloballyAvailable + , availability = pure $ + if islocal then LocallyAvailable else GloballyAvailable , remotetype = remote , mkUnavailable = return Nothing , getInfo = return [("url", url)] diff --git a/Remote/S3.hs b/Remote/S3.hs index b96fc96dcb..a2ab5a61ff 100644 --- a/Remote/S3.hs +++ b/Remote/S3.hs @@ -249,7 +249,7 @@ gen r u rc gc rs = do , readonly = False , appendonly = False , untrustworthy = False - , availability = GloballyAvailable + , availability = pure GloballyAvailable , remotetype = remote , mkUnavailable = gen r u (M.insert hostField (Proposed "!dne!") rc) gc rs , getInfo = includeCredsInfo c (AWS.creds u) (s3Info c info) diff --git a/Remote/Tahoe.hs b/Remote/Tahoe.hs index d5b6a0902e..7c2ad0c25a 100644 --- a/Remote/Tahoe.hs +++ b/Remote/Tahoe.hs @@ -108,7 +108,7 @@ gen r u rc gc rs = do , readonly = False , appendonly = False , untrustworthy = False - , availability = GloballyAvailable + , availability = pure GloballyAvailable , remotetype = remote , mkUnavailable = return Nothing , getInfo = return [] diff --git a/Remote/Web.hs b/Remote/Web.hs index 4a1b7a61c3..b3b788d2df 100644 --- a/Remote/Web.hs +++ b/Remote/Web.hs @@ -93,7 +93,7 @@ gen r u rc gc rs = do , readonly = True , appendonly = False , untrustworthy = False - , availability = GloballyAvailable + , availability = pure GloballyAvailable , remotetype = remote , mkUnavailable = return Nothing , getInfo = return [] diff --git a/Remote/WebDAV.hs b/Remote/WebDAV.hs index e9e771d754..08594dfba4 100644 --- a/Remote/WebDAV.hs +++ b/Remote/WebDAV.hs @@ -117,7 +117,7 @@ gen r u rc gc rs = do , readonly = False , appendonly = False , untrustworthy = False - , availability = GloballyAvailable + , availability = pure GloballyAvailable , remotetype = remote , mkUnavailable = gen r u (M.insert urlField (Proposed "http://!dne!/") rc) gc rs , getInfo = includeCredsInfo c (davCreds u) $ diff --git a/Types/Availability.hs b/Types/Availability.hs index 9e8bbf63b7..3d3e6ac416 100644 --- a/Types/Availability.hs +++ b/Types/Availability.hs @@ -7,5 +7,5 @@ module Types.Availability where -data Availability = GloballyAvailable | LocallyAvailable - deriving (Eq, Show, Read) +data Availability = GloballyAvailable | LocallyAvailable | Unavailable + deriving (Eq, Show) diff --git a/Types/GitConfig.hs b/Types/GitConfig.hs index 833db43c0f..13978e8338 100644 --- a/Types/GitConfig.hs +++ b/Types/GitConfig.hs @@ -37,7 +37,6 @@ import Utility.DataUnits import Config.Cost import Types.UUID import Types.Distribution -import Types.Availability import Types.Concurrency import Types.NumCopies import Types.Difference @@ -351,7 +350,6 @@ data RemoteGitConfig = RemoteGitConfig , remoteAnnexTrustLevel :: Maybe String , remoteAnnexStartCommand :: Maybe String , remoteAnnexStopCommand :: Maybe String - , remoteAnnexAvailability :: Maybe Availability , remoteAnnexSpeculatePresent :: Bool , remoteAnnexBare :: Maybe Bool , remoteAnnexRetry :: Maybe Integer @@ -416,7 +414,6 @@ extractRemoteGitConfig r remotename = do , remoteAnnexTrustLevel = notempty $ getmaybe "trustlevel" , remoteAnnexStartCommand = notempty $ getmaybe "start-command" , remoteAnnexStopCommand = notempty $ getmaybe "stop-command" - , remoteAnnexAvailability = getmayberead "availability" , remoteAnnexSpeculatePresent = getbool "speculate-present" False , remoteAnnexBare = getmaybebool "bare" , remoteAnnexRetry = getmayberead "retry" diff --git a/Types/Remote.hs b/Types/Remote.hs index 52b5adab6b..971c4a87c6 100644 --- a/Types/Remote.hs +++ b/Types/Remote.hs @@ -152,7 +152,8 @@ data RemoteA a = Remote -- decide. , untrustworthy :: Bool -- a Remote can be globally available. (Ie, "in the cloud".) - , availability :: Availability + -- Some Remotes can mark themselves unavailable. + , availability :: a Availability -- the type of the remote , remotetype :: RemoteTypeA a -- For testing, makes a version of this remote that is not diff --git a/doc/design/external_special_remote_protocol.mdwn b/doc/design/external_special_remote_protocol.mdwn index d4bf6bcf3d..26a4a51858 100644 --- a/doc/design/external_special_remote_protocol.mdwn +++ b/doc/design/external_special_remote_protocol.mdwn @@ -45,7 +45,7 @@ Recent versions of git-annex respond with a message indicating protocol extensions that it supports. Older versions of git-annex do not send this message. - EXTENSIONS INFO ASYNC GETGITREMOTENAME + EXTENSIONS INFO ASYNC GETGITREMOTENAME UNAVAILABLERESPONSE The special remote can respond to that with its own EXTENSIONS message, listing any extensions it wants to use. @@ -207,9 +207,19 @@ the special remote can reply with `UNSUPPORTED-REQUEST`. (Ie stored in the cloud vs on a local disk.) If the remote replies with `UNSUPPORTED-REQUEST`, its availability is assumed to be global. So, only remotes that are only reachable - locally need to worry about implementing this. + locally need to worry about implementing this. + This is queried at remote startup, so should avoid doing anything that + can take long to run or is expensive. Checking if a directory where the + remote stores files is currently mounted is the kind of thing it makes + sense to do here. * `AVAILABILITY GLOBAL|LOCAL` Indicates if the remote is globally or only locally available. + * `AVAILABILITY UNAVAILABLE` + Indicates that the remote is not currently available. + This will prevent some git-annex commands like `git-annex sync` from + trying to use the remote. + Older versions of git-annex do not support this response, so avoid + sending it unless the `UNAVAILABLERESPONSE` extension is enabled. * `CLAIMURL Url` Asks the remote if it wishes to claim responsibility for downloading an url. @@ -445,13 +455,16 @@ avoid talking to the buggy old version of git-annex. These protocol extensions are currently supported. * `INFO` - This makes the `INFO` message available to use. + This allows using the `INFO` message. * `ASYNC` This lets multiple actions be performed at the same time by a single external special remote program, rather than starting multiple programs. See the [[async_appendix]] for details. * `GETGITREMOTENAME` - This makes the `GETGITREMOTENAME` message available to use. + This allows using the `GETGITREMOTENAME` message. +* `UNAVAILABLERESPONSE` + This allows the `AVAILABILITY UNAVAILABLE` response to be used + in reply to `GETAVAILABILITY`. ## signals diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index d8b73cf0b6..b51e6dc881 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -1565,8 +1565,7 @@ Remotes are configured using these settings in `.git/config`. * `remote..annex-availability` - Can be used to tell git-annex whether a remote is LocallyAvailable - or GloballyAvailable. Normally, git-annex determines this automatically. + This configuration setting is no longer used. * `remote..annex-speculate-present`