diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs index 1dbb27fd4e..9be4558761 100644 --- a/Assistant/DaemonStatus.hs +++ b/Assistant/DaemonStatus.hs @@ -13,6 +13,7 @@ import Assistant.Common import Assistant.Alert.Utility import Utility.Tmp import Utility.NotificationBroadcaster +import Types.Availability import Types.Transfer import Logs.Transfer import Logs.Trust @@ -59,6 +60,7 @@ calcSyncRemotes = do let (exportremotes, nonexportremotes) = partition (exportTree . Remote.config) contentremotes let isimport r = importTree (Remote.config r) || Remote.thirdPartyPopulated (Remote.remotetype r) let dataremotes = filter (not . isimport) nonexportremotes + tocloud <- anyM iscloud contentremotes return $ \dstatus -> dstatus { syncRemotes = syncable @@ -66,10 +68,14 @@ calcSyncRemotes = do , syncDataRemotes = dataremotes , exportRemotes = exportremotes , downloadRemotes = contentremotes - , syncingToCloudRemote = any iscloud contentremotes + , syncingToCloudRemote = tocloud } where - iscloud r = not (Remote.readonly r) && Remote.availability r == Remote.GloballyAvailable + iscloud r + | Remote.readonly r = pure False + | otherwise = tryNonAsync (Remote.availability r) >>= return . \case + Right GloballyAvailable -> True + _ -> False {- Updates the syncRemotes list from the list of all remotes in Annex state. -} updateSyncRemotes :: Assistant () diff --git a/CHANGELOG b/CHANGELOG index 2282356058..56b6e138e9 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -8,6 +8,11 @@ git-annex (10.20230803) UNRELEASED; urgency=medium directory does not exist. An empty tree was imported, rather than the import failing. * Stop bundling curl in the OSX dmg and linux standalone image. + * sync, assist, push, pull: Skip more types of remotes when they + are not present due to eg being on a drive that is offline. + * Added AVAILABILITY UNAVAILABLE and the UNAVAILABLERESPONSE extension + to the external special remote protocol. + * The remote.name.annex-availability git config is no longer used. -- Joey Hess Mon, 07 Aug 2023 13:04:13 -0400 diff --git a/Command/Sync.hs b/Command/Sync.hs index 0d6e0c143f..44edf2142a 100644 --- a/Command/Sync.hs +++ b/Command/Sync.hs @@ -78,6 +78,7 @@ import Annex.Import import Annex.CheckIgnore import Types.FileMatcher import Types.GitConfig +import Types.Availability import qualified Database.Export as Export import Utility.Bloom import Utility.OptParse @@ -388,10 +389,9 @@ syncRemotes' ps available = listed = concat <$> mapM Remote.byNameOrGroup ps - good r - | Remote.gitSyncableRemoteType (Remote.remotetype r) = - Remote.Git.repoAvail =<< Remote.getRepo r - | otherwise = return True + good r = tryNonAsync (Remote.availability r) >>= return . \case + Right Unavailable -> False + _ -> True fastest = fromMaybe [] . headMaybe . Remote.byCost diff --git a/Remote/Adb.hs b/Remote/Adb.hs index f70fe27bdf..7d4e38696a 100644 --- a/Remote/Adb.hs +++ b/Remote/Adb.hs @@ -113,7 +113,7 @@ gen r u rc gc rs = do , gitconfig = gc , localpath = Nothing , remotetype = remote - , availability = LocallyAvailable + , availability = pure LocallyAvailable , readonly = False , appendonly = False , untrustworthy = False diff --git a/Remote/BitTorrent.hs b/Remote/BitTorrent.hs index dcc6f29004..ccc67a25d0 100644 --- a/Remote/BitTorrent.hs +++ b/Remote/BitTorrent.hs @@ -88,7 +88,7 @@ gen r _ rc gc rs = do , readonly = True , appendonly = False , untrustworthy = False - , availability = GloballyAvailable + , availability = pure GloballyAvailable , remotetype = remote , mkUnavailable = return Nothing , getInfo = return [] diff --git a/Remote/Borg.hs b/Remote/Borg.hs index d3b152c802..ba22c8cf59 100644 --- a/Remote/Borg.hs +++ b/Remote/Borg.hs @@ -112,7 +112,8 @@ gen r u rc gc rs = do , gitconfig = gc , localpath = borgRepoLocalPath borgrepo , remotetype = remote - , availability = if borgLocal borgrepo then LocallyAvailable else GloballyAvailable + , availability = pure $ + if borgLocal borgrepo then LocallyAvailable else GloballyAvailable , readonly = False , appendonly = False -- When the user sets the appendonly field, they are diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 8bcacf9b94..c754850e22 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -97,7 +97,8 @@ gen r u rc gc rs = do then Just buprepo else Nothing , remotetype = remote - , availability = if bupLocal buprepo then LocallyAvailable else GloballyAvailable + , availability = pure $ + if bupLocal buprepo then LocallyAvailable else GloballyAvailable , readonly = False , appendonly = False , untrustworthy = False diff --git a/Remote/Ddar.hs b/Remote/Ddar.hs index 3fc3fdbafb..375920a325 100644 --- a/Remote/Ddar.hs +++ b/Remote/Ddar.hs @@ -98,7 +98,8 @@ gen r u rc gc rs = do then Just $ ddarRepoLocation ddarrepo else Nothing , remotetype = remote - , availability = if ddarLocal ddarrepo then LocallyAvailable else GloballyAvailable + , availability = pure $ + if ddarLocal ddarrepo then LocallyAvailable else GloballyAvailable , readonly = False , appendonly = False , untrustworthy = False diff --git a/Remote/Directory.hs b/Remote/Directory.hs index d234ff59cb..b030bb2c46 100644 --- a/Remote/Directory.hs +++ b/Remote/Directory.hs @@ -134,7 +134,7 @@ gen r u rc gc rs = do , readonly = False , appendonly = False , untrustworthy = False - , availability = LocallyAvailable + , availability = pure LocallyAvailable , remotetype = remote , mkUnavailable = gen r u rc (gc { remoteAnnexDirectory = Just "/dev/null" }) rs diff --git a/Remote/External.hs b/Remote/External.hs index c429dc80fa..22a9a61d32 100644 --- a/Remote/External.hs +++ b/Remote/External.hs @@ -68,7 +68,7 @@ gen r u rc gc rs | externaltype == "readonly" = do c <- parsedRemoteConfig remote rc cst <- remoteCost gc c expensiveRemoteCost - let rmt = mk c cst GloballyAvailable + let rmt = mk c cst (pure GloballyAvailable) Nothing (externalInfo externaltype) Nothing @@ -87,7 +87,6 @@ gen r u rc gc rs (Git.remoteName r) (Just rs) Annex.addCleanupAction (RemoteCleanup u) $ stopExternal external cst <- getCost external r gc c - avail <- getAvailability external r gc exportsupported <- if exportTree c then checkExportSupported' external else return False @@ -107,7 +106,7 @@ gen r u rc gc rs let cheapexportsupported = if exportsupported then exportIsSupported else exportUnsupported - let rmt = mk c cst avail + let rmt = mk c cst (getAvailability external) (Just (whereisKeyM external)) (getInfoM external) (Just (claimUrlM external)) @@ -777,25 +776,16 @@ getCost external r gc pc = return c defcst = expensiveRemoteCost -{- Caches the availability in the git config to avoid needing to start up an - - external special remote every time time just to ask it what its - - availability is. - - - - Most remotes do not bother to implement a reply to this request; +{- Most remotes do not bother to implement a reply to this request; - globally available is the default. -} -getAvailability :: External -> Git.Repo -> RemoteGitConfig -> Annex Availability -getAvailability external r gc = - maybe (catchNonAsync query (const (pure defavail))) return - (remoteAnnexAvailability gc) +getAvailability :: External -> Annex Availability +getAvailability external = catchNonAsync query (const (pure defavail)) where - query = do - avail <- handleRequest external GETAVAILABILITY Nothing $ \req -> case req of - AVAILABILITY avail -> result avail - UNSUPPORTED_REQUEST -> result defavail - _ -> Nothing - setRemoteAvailability r avail - return avail + query = handleRequest external GETAVAILABILITY Nothing $ \req -> case req of + AVAILABILITY avail -> result avail + UNSUPPORTED_REQUEST -> result defavail + _ -> Nothing defavail = GloballyAvailable claimUrlM :: External -> URLString -> Annex Bool diff --git a/Remote/External/Types.hs b/Remote/External/Types.hs index ed8aa20346..1ee29ebd6c 100644 --- a/Remote/External/Types.hs +++ b/Remote/External/Types.hs @@ -110,6 +110,7 @@ supportedExtensionList :: ExtensionList supportedExtensionList = ExtensionList [ "INFO" , "GETGITREMOTENAME" + , "UNAVAILABLERESPONSE" , asyncExtension ] @@ -447,9 +448,11 @@ instance Proto.Serializable Size where instance Proto.Serializable Availability where serialize GloballyAvailable = "GLOBAL" serialize LocallyAvailable = "LOCAL" + serialize Unavailable = "UNAVAILABLE" deserialize "GLOBAL" = Just GloballyAvailable deserialize "LOCAL" = Just LocallyAvailable + deserialize "UNAVAILABLE" = Just Unavailable deserialize _ = Nothing instance Proto.Serializable [(URLString, Size, FilePath)] where diff --git a/Remote/GCrypt.hs b/Remote/GCrypt.hs index 61e8b7210e..bf8f01ed3e 100644 --- a/Remote/GCrypt.hs +++ b/Remote/GCrypt.hs @@ -158,7 +158,7 @@ gen' r u c gc rs = do , readonly = Git.repoIsHttp r , appendonly = False , untrustworthy = False - , availability = availabilityCalc r + , availability = pure (availabilityCalc r) , remotetype = remote , mkUnavailable = return Nothing , getInfo = gitRepoInfo this diff --git a/Remote/Git.hs b/Remote/Git.hs index 387fb9d4f0..d4708e40fa 100644 --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -1,6 +1,6 @@ {- Standard git remotes. - - - Copyright 2011-2021 Joey Hess + - Copyright 2011-2023 Joey Hess - - Licensed under the GNU AGPL version 3 or higher. -} @@ -11,7 +11,6 @@ module Remote.Git ( remote, configRead, - repoAvail, onLocalRepo, ) where @@ -210,7 +209,7 @@ gen r u rc gc rs , readonly = Git.repoIsHttp r , appendonly = False , untrustworthy = False - , availability = availabilityCalc r + , availability = repoAvail r , remotetype = remote , mkUnavailable = unavailable r u rc gc rs , getInfo = gitRepoInfo new @@ -233,20 +232,24 @@ unavailable r = gen r' _ -> r -- already unavailable {- Checks relatively inexpensively if a repository is available for use. -} -repoAvail :: Git.Repo -> Annex Bool +repoAvail :: Git.Repo -> Annex Availability repoAvail r - | Git.repoIsHttp r = return True + | Git.repoIsHttp r = return GloballyAvailable | Git.GCrypt.isEncrypted r = do g <- gitRepo liftIO $ do er <- Git.GCrypt.encryptedRemote g r if Git.repoIsLocal er || Git.repoIsLocalUnknown er - then catchBoolIO $ - void (Git.Config.read er) >> return True - else return True - | Git.repoIsUrl r = return True - | Git.repoIsLocalUnknown r = return False - | otherwise = liftIO $ isJust <$> catchMaybeIO (Git.Config.read r) + then checklocal er + else return GloballyAvailable + | Git.repoIsUrl r = return GloballyAvailable + | Git.repoIsLocalUnknown r = return Unavailable + | otherwise = checklocal r + where + checklocal r' = ifM (liftIO $ isJust <$> catchMaybeIO (Git.Config.read r')) + ( return LocallyAvailable + , return Unavailable + ) {- Tries to read the config for a specified remote, updates state, and - returns the updated repo. -} diff --git a/Remote/GitLFS.hs b/Remote/GitLFS.hs index fc3459f46f..31ddc6ccef 100644 --- a/Remote/GitLFS.hs +++ b/Remote/GitLFS.hs @@ -130,7 +130,7 @@ gen r u rc gc rs = do , gitconfig = gc , localpath = Nothing , remotetype = remote - , availability = GloballyAvailable + , availability = pure GloballyAvailable , readonly = False -- content cannot be removed from a git-lfs repo , appendonly = True diff --git a/Remote/Glacier.hs b/Remote/Glacier.hs index e97ad12da4..18bd65f21b 100644 --- a/Remote/Glacier.hs +++ b/Remote/Glacier.hs @@ -103,7 +103,7 @@ gen r u rc gc rs = do , readonly = False , appendonly = False , untrustworthy = False - , availability = GloballyAvailable + , availability = pure GloballyAvailable , remotetype = remote , mkUnavailable = return Nothing , getInfo = includeCredsInfo c (AWS.creds u) $ diff --git a/Remote/Hook.hs b/Remote/Hook.hs index 49d72a8a6c..a7dd3e6593 100644 --- a/Remote/Hook.hs +++ b/Remote/Hook.hs @@ -82,7 +82,7 @@ gen r u rc gc rs = do , readonly = False , appendonly = False , untrustworthy = False - , availability = GloballyAvailable + , availability = pure GloballyAvailable , remotetype = remote , mkUnavailable = gen r u rc (gc { remoteAnnexHookType = Just "!dne!" }) diff --git a/Remote/HttpAlso.hs b/Remote/HttpAlso.hs index ff5a638ced..06003b7b54 100644 --- a/Remote/HttpAlso.hs +++ b/Remote/HttpAlso.hs @@ -95,7 +95,7 @@ gen r u rc gc rs = do , readonly = True , appendonly = False , untrustworthy = False - , availability = GloballyAvailable + , availability = pure GloballyAvailable , remotetype = remote , mkUnavailable = return Nothing , getInfo = return [] diff --git a/Remote/P2P.hs b/Remote/P2P.hs index a5fd926a36..4b47e2f489 100644 --- a/Remote/P2P.hs +++ b/Remote/P2P.hs @@ -77,7 +77,7 @@ chainGen addr r u rc gc rs = do , readonly = False , appendonly = False , untrustworthy = False - , availability = GloballyAvailable + , availability = pure GloballyAvailable , remotetype = remote , mkUnavailable = return Nothing , getInfo = gitRepoInfo this diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs index c256fe6784..b0fceeab24 100644 --- a/Remote/Rsync.hs +++ b/Remote/Rsync.hs @@ -120,7 +120,8 @@ gen r u rc gc rs = do , readonly = False , appendonly = False , untrustworthy = False - , availability = if islocal then LocallyAvailable else GloballyAvailable + , availability = pure $ + if islocal then LocallyAvailable else GloballyAvailable , remotetype = remote , mkUnavailable = return Nothing , getInfo = return [("url", url)] diff --git a/Remote/S3.hs b/Remote/S3.hs index b96fc96dcb..a2ab5a61ff 100644 --- a/Remote/S3.hs +++ b/Remote/S3.hs @@ -249,7 +249,7 @@ gen r u rc gc rs = do , readonly = False , appendonly = False , untrustworthy = False - , availability = GloballyAvailable + , availability = pure GloballyAvailable , remotetype = remote , mkUnavailable = gen r u (M.insert hostField (Proposed "!dne!") rc) gc rs , getInfo = includeCredsInfo c (AWS.creds u) (s3Info c info) diff --git a/Remote/Tahoe.hs b/Remote/Tahoe.hs index d5b6a0902e..7c2ad0c25a 100644 --- a/Remote/Tahoe.hs +++ b/Remote/Tahoe.hs @@ -108,7 +108,7 @@ gen r u rc gc rs = do , readonly = False , appendonly = False , untrustworthy = False - , availability = GloballyAvailable + , availability = pure GloballyAvailable , remotetype = remote , mkUnavailable = return Nothing , getInfo = return [] diff --git a/Remote/Web.hs b/Remote/Web.hs index 4a1b7a61c3..b3b788d2df 100644 --- a/Remote/Web.hs +++ b/Remote/Web.hs @@ -93,7 +93,7 @@ gen r u rc gc rs = do , readonly = True , appendonly = False , untrustworthy = False - , availability = GloballyAvailable + , availability = pure GloballyAvailable , remotetype = remote , mkUnavailable = return Nothing , getInfo = return [] diff --git a/Remote/WebDAV.hs b/Remote/WebDAV.hs index e9e771d754..08594dfba4 100644 --- a/Remote/WebDAV.hs +++ b/Remote/WebDAV.hs @@ -117,7 +117,7 @@ gen r u rc gc rs = do , readonly = False , appendonly = False , untrustworthy = False - , availability = GloballyAvailable + , availability = pure GloballyAvailable , remotetype = remote , mkUnavailable = gen r u (M.insert urlField (Proposed "http://!dne!/") rc) gc rs , getInfo = includeCredsInfo c (davCreds u) $ diff --git a/Types/Availability.hs b/Types/Availability.hs index 9e8bbf63b7..3d3e6ac416 100644 --- a/Types/Availability.hs +++ b/Types/Availability.hs @@ -7,5 +7,5 @@ module Types.Availability where -data Availability = GloballyAvailable | LocallyAvailable - deriving (Eq, Show, Read) +data Availability = GloballyAvailable | LocallyAvailable | Unavailable + deriving (Eq, Show) diff --git a/Types/GitConfig.hs b/Types/GitConfig.hs index 833db43c0f..13978e8338 100644 --- a/Types/GitConfig.hs +++ b/Types/GitConfig.hs @@ -37,7 +37,6 @@ import Utility.DataUnits import Config.Cost import Types.UUID import Types.Distribution -import Types.Availability import Types.Concurrency import Types.NumCopies import Types.Difference @@ -351,7 +350,6 @@ data RemoteGitConfig = RemoteGitConfig , remoteAnnexTrustLevel :: Maybe String , remoteAnnexStartCommand :: Maybe String , remoteAnnexStopCommand :: Maybe String - , remoteAnnexAvailability :: Maybe Availability , remoteAnnexSpeculatePresent :: Bool , remoteAnnexBare :: Maybe Bool , remoteAnnexRetry :: Maybe Integer @@ -416,7 +414,6 @@ extractRemoteGitConfig r remotename = do , remoteAnnexTrustLevel = notempty $ getmaybe "trustlevel" , remoteAnnexStartCommand = notempty $ getmaybe "start-command" , remoteAnnexStopCommand = notempty $ getmaybe "stop-command" - , remoteAnnexAvailability = getmayberead "availability" , remoteAnnexSpeculatePresent = getbool "speculate-present" False , remoteAnnexBare = getmaybebool "bare" , remoteAnnexRetry = getmayberead "retry" diff --git a/Types/Remote.hs b/Types/Remote.hs index 52b5adab6b..971c4a87c6 100644 --- a/Types/Remote.hs +++ b/Types/Remote.hs @@ -152,7 +152,8 @@ data RemoteA a = Remote -- decide. , untrustworthy :: Bool -- a Remote can be globally available. (Ie, "in the cloud".) - , availability :: Availability + -- Some Remotes can mark themselves unavailable. + , availability :: a Availability -- the type of the remote , remotetype :: RemoteTypeA a -- For testing, makes a version of this remote that is not diff --git a/doc/design/external_special_remote_protocol.mdwn b/doc/design/external_special_remote_protocol.mdwn index d4bf6bcf3d..26a4a51858 100644 --- a/doc/design/external_special_remote_protocol.mdwn +++ b/doc/design/external_special_remote_protocol.mdwn @@ -45,7 +45,7 @@ Recent versions of git-annex respond with a message indicating protocol extensions that it supports. Older versions of git-annex do not send this message. - EXTENSIONS INFO ASYNC GETGITREMOTENAME + EXTENSIONS INFO ASYNC GETGITREMOTENAME UNAVAILABLERESPONSE The special remote can respond to that with its own EXTENSIONS message, listing any extensions it wants to use. @@ -207,9 +207,19 @@ the special remote can reply with `UNSUPPORTED-REQUEST`. (Ie stored in the cloud vs on a local disk.) If the remote replies with `UNSUPPORTED-REQUEST`, its availability is assumed to be global. So, only remotes that are only reachable - locally need to worry about implementing this. + locally need to worry about implementing this. + This is queried at remote startup, so should avoid doing anything that + can take long to run or is expensive. Checking if a directory where the + remote stores files is currently mounted is the kind of thing it makes + sense to do here. * `AVAILABILITY GLOBAL|LOCAL` Indicates if the remote is globally or only locally available. + * `AVAILABILITY UNAVAILABLE` + Indicates that the remote is not currently available. + This will prevent some git-annex commands like `git-annex sync` from + trying to use the remote. + Older versions of git-annex do not support this response, so avoid + sending it unless the `UNAVAILABLERESPONSE` extension is enabled. * `CLAIMURL Url` Asks the remote if it wishes to claim responsibility for downloading an url. @@ -445,13 +455,16 @@ avoid talking to the buggy old version of git-annex. These protocol extensions are currently supported. * `INFO` - This makes the `INFO` message available to use. + This allows using the `INFO` message. * `ASYNC` This lets multiple actions be performed at the same time by a single external special remote program, rather than starting multiple programs. See the [[async_appendix]] for details. * `GETGITREMOTENAME` - This makes the `GETGITREMOTENAME` message available to use. + This allows using the `GETGITREMOTENAME` message. +* `UNAVAILABLERESPONSE` + This allows the `AVAILABILITY UNAVAILABLE` response to be used + in reply to `GETAVAILABILITY`. ## signals diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index d8b73cf0b6..b51e6dc881 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -1565,8 +1565,7 @@ Remotes are configured using these settings in `.git/config`. * `remote..annex-availability` - Can be used to tell git-annex whether a remote is LocallyAvailable - or GloballyAvailable. Normally, git-annex determines this automatically. + This configuration setting is no longer used. * `remote..annex-speculate-present`