let Remote.availability return Unavilable

This is groundwork for making special remotes like borg be skipped by
sync when on an offline drive.

Added AVAILABILITY UNAVAILABLE reponse and the UNAVAILABLERESPONSE extension
to the external special remote protocol. The extension is needed because
old git-annex, if it sees that response, will display a warning
message. (It does continue as if the remote is globally available, which
is acceptable, and the warning is only displayed at initremote due to
remote.name.annex-availability caching, but still it seemed best to make
this a protocol extension.)

The remote.name.annex-availability git config is no longer used any
more, and is documented as such. It was only used by external special
remotes to cache the availability, to avoid needing to start the
external process every time. Now that availability is queried as an
Annex action, the external is only started by sync (and the assistant),
when they actually check availability.

Sponsored-by: Nicholas Golder-Manning on Patreon

Assistant/DaemonStatus.hs

@@ -13,6 +13,7 @@ import Assistant.Common
 import Assistant.Alert.Utility
 import Utility.Tmp
 import Utility.NotificationBroadcaster
+import Types.Availability
 import Types.Transfer
 import Logs.Transfer
 import Logs.Trust
@@ -59,6 +60,7 @@ calcSyncRemotes = do
 	let (exportremotes, nonexportremotes) = partition (exportTree . Remote.config) contentremotes
 	let isimport r = importTree (Remote.config r) || Remote.thirdPartyPopulated (Remote.remotetype r)
 	let dataremotes = filter (not . isimport) nonexportremotes
+	tocloud <- anyM iscloud contentremotes
 
 	return $ \dstatus -> dstatus
 		{ syncRemotes = syncable
@@ -66,10 +68,14 @@ calcSyncRemotes = do
 		, syncDataRemotes = dataremotes
 		, exportRemotes = exportremotes
 		, downloadRemotes = contentremotes
-		, syncingToCloudRemote = any iscloud contentremotes
+		, syncingToCloudRemote = tocloud
 		}
   where
-	iscloud r = not (Remote.readonly r) && Remote.availability r == Remote.GloballyAvailable
+	iscloud r
+		| Remote.readonly r = pure False
+		| otherwise = tryNonAsync (Remote.availability r) >>= return . \case
+			Right GloballyAvailable -> True
+			_ -> False
 
 {- Updates the syncRemotes list from the list of all remotes in Annex state. -}
 updateSyncRemotes :: Assistant ()

CHANGELOG

@@ -8,6 +8,11 @@ git-annex (10.20230803) UNRELEASED; urgency=medium
     directory does not exist. An empty tree was imported, rather than the
     import failing.
   * Stop bundling curl in the OSX dmg and linux standalone image.
+  * sync, assist, push, pull: Skip more types of remotes when they
+    are not present due to eg being on a drive that is offline.
+  * Added AVAILABILITY UNAVAILABLE and the UNAVAILABLERESPONSE extension
+    to the external special remote protocol.
+  * The remote.name.annex-availability git config is no longer used.
 
  -- Joey Hess <id@joeyh.name>  Mon, 07 Aug 2023 13:04:13 -0400
 

Command/Sync.hs

@@ -78,6 +78,7 @@ import Annex.Import
 import Annex.CheckIgnore
 import Types.FileMatcher
 import Types.GitConfig
+import Types.Availability
 import qualified Database.Export as Export
 import Utility.Bloom
 import Utility.OptParse
@@ -388,10 +389,9 @@ syncRemotes' ps available =
 	
 	listed = concat <$> mapM Remote.byNameOrGroup ps
 	
-	good r
+	good r = tryNonAsync (Remote.availability r) >>= return . \case
-		| Remote.gitSyncableRemoteType (Remote.remotetype r) =
+		Right Unavailable -> False
-			Remote.Git.repoAvail =<< Remote.getRepo r
+		_ -> True
-		| otherwise = return True
 	
 	fastest = fromMaybe [] . headMaybe . Remote.byCost
 

Remote/Adb.hs

@@ -113,7 +113,7 @@ gen r u rc gc rs = do
 		, gitconfig = gc
 		, localpath = Nothing
 		, remotetype = remote
-		, availability = LocallyAvailable
+		, availability = pure LocallyAvailable
 		, readonly = False
 		, appendonly = False
 		, untrustworthy = False

Remote/BitTorrent.hs

@@ -88,7 +88,7 @@ gen r _ rc gc rs = do
 		, readonly = True
 		, appendonly = False
 		, untrustworthy = False
-		, availability = GloballyAvailable
+		, availability = pure GloballyAvailable
 		, remotetype = remote
 		, mkUnavailable = return Nothing
 		, getInfo = return []

Remote/Borg.hs

@@ -112,7 +112,8 @@ gen r u rc gc rs = do
 		, gitconfig = gc
 		, localpath = borgRepoLocalPath borgrepo
 		, remotetype = remote
-		, availability = if borgLocal borgrepo then LocallyAvailable else GloballyAvailable
+		, availability = pure $
+			if borgLocal borgrepo then LocallyAvailable else GloballyAvailable
 		, readonly = False
 		, appendonly = False
 		-- When the user sets the appendonly field, they are

Remote/Bup.hs

@@ -97,7 +97,8 @@ gen r u rc gc rs = do
 			then Just buprepo
 			else Nothing
 		, remotetype = remote
-		, availability = if bupLocal buprepo then LocallyAvailable else GloballyAvailable
+		, availability = pure $
+			if bupLocal buprepo then LocallyAvailable else GloballyAvailable
 		, readonly = False
 		, appendonly = False
 		, untrustworthy = False

Remote/Ddar.hs

@@ -98,7 +98,8 @@ gen r u rc gc rs = do
 			then Just $ ddarRepoLocation ddarrepo
 			else Nothing
 		, remotetype = remote
-		, availability = if ddarLocal ddarrepo then LocallyAvailable else GloballyAvailable
+		, availability = pure $
+			if ddarLocal ddarrepo then LocallyAvailable else GloballyAvailable
 		, readonly = False
 		, appendonly = False
 		, untrustworthy = False

Remote/Directory.hs

@@ -134,7 +134,7 @@ gen r u rc gc rs = do
 			, readonly = False
 			, appendonly = False
 			, untrustworthy = False
-			, availability = LocallyAvailable
+			, availability = pure LocallyAvailable
 			, remotetype = remote
 			, mkUnavailable = gen r u rc
 				(gc { remoteAnnexDirectory = Just "/dev/null" }) rs

Remote/External.hs

@@ -68,7 +68,7 @@ gen r u rc gc rs
 	| externaltype == "readonly" = do
 		c <- parsedRemoteConfig remote rc
 		cst <- remoteCost gc c expensiveRemoteCost
-		let rmt = mk c cst GloballyAvailable
+		let rmt = mk c cst (pure GloballyAvailable)
 			Nothing
 			(externalInfo externaltype)
 			Nothing
@@ -87,7 +87,6 @@ gen r u rc gc rs
 			(Git.remoteName r) (Just rs)
 		Annex.addCleanupAction (RemoteCleanup u) $ stopExternal external
 		cst <- getCost external r gc c
-		avail <- getAvailability external r gc
 		exportsupported <- if exportTree c
 			then checkExportSupported' external
 			else return False
@@ -107,7 +106,7 @@ gen r u rc gc rs
 		let cheapexportsupported = if exportsupported
 			then exportIsSupported
 			else exportUnsupported
-		let rmt = mk c cst avail
+		let rmt = mk c cst (getAvailability external)
 			(Just (whereisKeyM external))
 			(getInfoM external)
 			(Just (claimUrlM external))
@@ -777,25 +776,16 @@ getCost external r gc pc =
 		return c
 	defcst = expensiveRemoteCost
 
-{- Caches the availability in the git config to avoid needing to start up an
+{- Most remotes do not bother to implement a reply to this request;
- - external special remote every time time just to ask it what its
- - availability is.
- -
- - Most remotes do not bother to implement a reply to this request;
  - globally available is the default.
  -}
-getAvailability :: External -> Git.Repo -> RemoteGitConfig -> Annex Availability
+getAvailability :: External -> Annex Availability
-getAvailability external r gc = 
+getAvailability external = catchNonAsync query (const (pure defavail))
-	maybe (catchNonAsync query (const (pure defavail))) return
-		(remoteAnnexAvailability gc)
   where
-	query = do
+	query = handleRequest external GETAVAILABILITY Nothing $ \req -> case req of
-		avail <- handleRequest external GETAVAILABILITY Nothing $ \req -> case req of
 		AVAILABILITY avail -> result avail
 		UNSUPPORTED_REQUEST -> result defavail
 		_ -> Nothing
-		setRemoteAvailability r avail
-		return avail
 	defavail = GloballyAvailable
 
 claimUrlM :: External -> URLString -> Annex Bool

Remote/External/Types.hs

@@ -110,6 +110,7 @@ supportedExtensionList :: ExtensionList
 supportedExtensionList = ExtensionList
 	[ "INFO"
 	, "GETGITREMOTENAME"
+	, "UNAVAILABLERESPONSE"
 	, asyncExtension
 	]
 
@@ -447,9 +448,11 @@ instance Proto.Serializable Size where
 instance Proto.Serializable Availability where
 	serialize GloballyAvailable = "GLOBAL"
 	serialize LocallyAvailable = "LOCAL"
+	serialize Unavailable = "UNAVAILABLE"
 
 	deserialize "GLOBAL" = Just GloballyAvailable
 	deserialize "LOCAL" = Just LocallyAvailable
+	deserialize "UNAVAILABLE" = Just Unavailable
 	deserialize _ = Nothing
 
 instance Proto.Serializable [(URLString, Size, FilePath)] where

Remote/GCrypt.hs

@@ -158,7 +158,7 @@ gen' r u c gc rs = do
 		, readonly = Git.repoIsHttp r
 		, appendonly = False
 		, untrustworthy = False
-		, availability = availabilityCalc r
+		, availability = pure (availabilityCalc r)
 		, remotetype = remote
 		, mkUnavailable = return Nothing
 		, getInfo = gitRepoInfo this

Remote/Git.hs

@@ -1,6 +1,6 @@
 {- Standard git remotes.
  -
- - Copyright 2011-2021 Joey Hess <id@joeyh.name>
+ - Copyright 2011-2023 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
@@ -11,7 +11,6 @@
 module Remote.Git (
 	remote,
 	configRead,
-	repoAvail,
 	onLocalRepo,
 ) where
 
@@ -210,7 +209,7 @@ gen r u rc gc rs
 			, readonly = Git.repoIsHttp r
 			, appendonly = False
 			, untrustworthy = False
-			, availability = availabilityCalc r
+			, availability = repoAvail r
 			, remotetype = remote
 			, mkUnavailable = unavailable r u rc gc rs
 			, getInfo = gitRepoInfo new
@@ -233,20 +232,24 @@ unavailable r = gen r'
 		_ -> r -- already unavailable
 
 {- Checks relatively inexpensively if a repository is available for use. -}
-repoAvail :: Git.Repo -> Annex Bool
+repoAvail :: Git.Repo -> Annex Availability
 repoAvail r 
-	| Git.repoIsHttp r = return True
+	| Git.repoIsHttp r = return GloballyAvailable
 	| Git.GCrypt.isEncrypted r = do
 		g <- gitRepo
 		liftIO $ do
 			er <- Git.GCrypt.encryptedRemote g r
 			if Git.repoIsLocal er || Git.repoIsLocalUnknown er
-				then catchBoolIO $
+				then checklocal er
-					void (Git.Config.read er) >> return True
+				else return GloballyAvailable
-				else return True
+	| Git.repoIsUrl r = return GloballyAvailable
-	| Git.repoIsUrl r = return True
+	| Git.repoIsLocalUnknown r = return Unavailable
-	| Git.repoIsLocalUnknown r = return False
+	| otherwise = checklocal r
-	| otherwise = liftIO $ isJust <$> catchMaybeIO (Git.Config.read r)
+  where
+	checklocal r' = ifM (liftIO $ isJust <$> catchMaybeIO (Git.Config.read r'))
+		( return LocallyAvailable
+		, return Unavailable
+		)
 
 {- Tries to read the config for a specified remote, updates state, and
  - returns the updated repo. -}

Remote/GitLFS.hs

@@ -130,7 +130,7 @@ gen r u rc gc rs = do
 		, gitconfig = gc
 		, localpath = Nothing
 		, remotetype = remote
-		, availability = GloballyAvailable
+		, availability = pure GloballyAvailable
 		, readonly = False
 		-- content cannot be removed from a git-lfs repo
 		, appendonly = True

Remote/Glacier.hs

@@ -103,7 +103,7 @@ gen r u rc gc rs = do
 			, readonly = False
 			, appendonly = False
 			, untrustworthy = False
-			, availability = GloballyAvailable
+			, availability = pure GloballyAvailable
 			, remotetype = remote
 			, mkUnavailable = return Nothing
 			, getInfo = includeCredsInfo c (AWS.creds u) $

Remote/Hook.hs

@@ -82,7 +82,7 @@ gen r u rc gc rs = do
 			, readonly = False
 			, appendonly = False
 			, untrustworthy = False
-			, availability = GloballyAvailable
+			, availability = pure GloballyAvailable
 			, remotetype = remote
 			, mkUnavailable = gen r u rc
 				(gc { remoteAnnexHookType = Just "!dne!" })

Remote/HttpAlso.hs

@@ -95,7 +95,7 @@ gen r u rc gc rs = do
 		, readonly = True
 		, appendonly = False
 		, untrustworthy = False
-		, availability = GloballyAvailable
+		, availability = pure GloballyAvailable
 		, remotetype = remote
 		, mkUnavailable = return Nothing
 		, getInfo = return []

Remote/P2P.hs

@@ -77,7 +77,7 @@ chainGen addr r u rc gc rs = do
 		, readonly = False
 		, appendonly = False
 		, untrustworthy = False
-		, availability = GloballyAvailable
+		, availability = pure GloballyAvailable
 		, remotetype = remote
 		, mkUnavailable = return Nothing
 		, getInfo = gitRepoInfo this

Remote/Rsync.hs

@@ -120,7 +120,8 @@ gen r u rc gc rs = do
 			, readonly = False
 			, appendonly = False
 			, untrustworthy = False
-			, availability = if islocal then LocallyAvailable else GloballyAvailable
+			, availability = pure $
+				if islocal then LocallyAvailable else GloballyAvailable
 			, remotetype = remote
 			, mkUnavailable = return Nothing
 			, getInfo = return [("url", url)]

Remote/S3.hs

@@ -249,7 +249,7 @@ gen r u rc gc rs = do
 			, readonly = False
 			, appendonly = False
 			, untrustworthy = False
-			, availability = GloballyAvailable
+			, availability = pure GloballyAvailable
 			, remotetype = remote
 			, mkUnavailable = gen r u (M.insert hostField (Proposed "!dne!") rc) gc rs
 			, getInfo = includeCredsInfo c (AWS.creds u) (s3Info c info)

Remote/Tahoe.hs

@@ -108,7 +108,7 @@ gen r u rc gc rs = do
 		, readonly = False
 		, appendonly = False
 		, untrustworthy = False
-		, availability = GloballyAvailable
+		, availability = pure GloballyAvailable
 		, remotetype = remote
 		, mkUnavailable = return Nothing
 		, getInfo = return []

Remote/Web.hs

@@ -93,7 +93,7 @@ gen r u rc gc rs = do
 		, readonly = True
 		, appendonly = False
 		, untrustworthy = False
-		, availability = GloballyAvailable
+		, availability = pure GloballyAvailable
 		, remotetype = remote
 		, mkUnavailable = return Nothing
 		, getInfo = return []

Remote/WebDAV.hs

@@ -117,7 +117,7 @@ gen r u rc gc rs = do
 			, readonly = False
 			, appendonly = False
 			, untrustworthy = False
-			, availability = GloballyAvailable
+			, availability = pure GloballyAvailable
 			, remotetype = remote
 			, mkUnavailable = gen r u (M.insert urlField (Proposed "http://!dne!/") rc) gc rs
 			, getInfo = includeCredsInfo c (davCreds u) $

Types/Availability.hs

@@ -7,5 +7,5 @@
 
 module Types.Availability where
 
-data Availability = GloballyAvailable | LocallyAvailable
+data Availability = GloballyAvailable | LocallyAvailable | Unavailable
-	deriving (Eq, Show, Read)
+	deriving (Eq, Show)

Types/GitConfig.hs

@@ -37,7 +37,6 @@ import Utility.DataUnits
 import Config.Cost
 import Types.UUID
 import Types.Distribution
-import Types.Availability
 import Types.Concurrency
 import Types.NumCopies
 import Types.Difference
@@ -351,7 +350,6 @@ data RemoteGitConfig = RemoteGitConfig
 	, remoteAnnexTrustLevel :: Maybe String
 	, remoteAnnexStartCommand :: Maybe String
 	, remoteAnnexStopCommand :: Maybe String
-	, remoteAnnexAvailability :: Maybe Availability
 	, remoteAnnexSpeculatePresent :: Bool
 	, remoteAnnexBare :: Maybe Bool
 	, remoteAnnexRetry :: Maybe Integer
@@ -416,7 +414,6 @@ extractRemoteGitConfig r remotename = do
 		, remoteAnnexTrustLevel = notempty $ getmaybe "trustlevel"
 		, remoteAnnexStartCommand = notempty $ getmaybe "start-command"
 		, remoteAnnexStopCommand = notempty $ getmaybe "stop-command"
-		, remoteAnnexAvailability = getmayberead "availability"
 		, remoteAnnexSpeculatePresent = getbool "speculate-present" False
 		, remoteAnnexBare = getmaybebool "bare"
 		, remoteAnnexRetry = getmayberead "retry"

Types/Remote.hs

@@ -152,7 +152,8 @@ data RemoteA a = Remote
 	-- decide.
 	, untrustworthy :: Bool
 	-- a Remote can be globally available. (Ie, "in the cloud".)
-	, availability :: Availability
+	-- Some Remotes can mark themselves unavailable.
+	, availability :: a Availability
 	-- the type of the remote
 	, remotetype :: RemoteTypeA a
 	-- For testing, makes a version of this remote that is not

doc/design/external_special_remote_protocol.mdwn

@@ -45,7 +45,7 @@ Recent versions of git-annex respond with a message indicating
 protocol extensions that it supports. Older versions of
 git-annex do not send this message.
 
-	EXTENSIONS INFO ASYNC GETGITREMOTENAME
+	EXTENSIONS INFO ASYNC GETGITREMOTENAME UNAVAILABLERESPONSE
 
 The special remote can respond to that with its own EXTENSIONS message, listing
 any extensions it wants to use. 
@@ -208,8 +208,18 @@ the special remote can reply with `UNSUPPORTED-REQUEST`.
   If the remote replies with `UNSUPPORTED-REQUEST`, its availability
   is assumed to be global. So, only remotes that are only reachable
   locally need to worry about implementing this.  
+  This is queried at remote startup, so should avoid doing anything that
+  can take long to run or is expensive. Checking if a directory where the
+  remote stores files is currently mounted is the kind of thing it makes
+  sense to do here.
   * `AVAILABILITY GLOBAL|LOCAL`  
     Indicates if the remote is globally or only locally available.
+  * `AVAILABILITY UNAVAILABLE`  
+    Indicates that the remote is not currently available.  
+    This will prevent some git-annex commands like `git-annex sync` from
+    trying to use the remote.  
+    Older versions of git-annex do not support this response, so avoid
+    sending it unless the `UNAVAILABLERESPONSE` extension is enabled.
 * `CLAIMURL Url`  
   Asks the remote if it wishes to claim responsibility for downloading
   an url.
@@ -445,13 +455,16 @@ avoid talking to the buggy old version of git-annex.
 These protocol extensions are currently supported.
 
 * `INFO`  
-  This makes the `INFO` message available to use.
+  This allows using the `INFO` message.
 * `ASYNC`  
   This lets multiple actions be performed at the same time by
   a single external special remote program, rather than starting multiple
   programs. See the [[async_appendix]] for details.
 * `GETGITREMOTENAME`  
-  This makes the `GETGITREMOTENAME` message available to use.
+  This allows using the `GETGITREMOTENAME` message.
+* `UNAVAILABLERESPONSE`  
+  This allows the `AVAILABILITY UNAVAILABLE` response to be used
+  in reply to `GETAVAILABILITY`.
 
 ## signals
 

doc/git-annex.mdwn

@@ -1565,8 +1565,7 @@ Remotes are configured using these settings in `.git/config`.
 
 * `remote.<name>.annex-availability`
 
-  Can be used to tell git-annex whether a remote is LocallyAvailable
+  This configuration setting is no longer used.
-  or GloballyAvailable. Normally, git-annex determines this automatically.
 
 * `remote.<name>.annex-speculate-present`