mirrors/git-annex
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

let Remote.availability return Unavilable

Browse source 
This is groundwork for making special remotes like borg be skipped by
sync when on an offline drive.

Added AVAILABILITY UNAVAILABLE reponse and the UNAVAILABLERESPONSE extension
to the external special remote protocol. The extension is needed because
old git-annex, if it sees that response, will display a warning
message. (It does continue as if the remote is globally available, which
is acceptable, and the warning is only displayed at initremote due to
remote.name.annex-availability caching, but still it seemed best to make
this a protocol extension.)

The remote.name.annex-availability git config is no longer used any
more, and is documented as such. It was only used by external special
remotes to cache the availability, to avoid needing to start the
external process every time. Now that availability is queried as an
Annex action, the external is only started by sync (and the assistant),
when they actually check availability.

Sponsored-by: Nicholas Golder-Manning on Patreon
This commit is contained in:
Joey Hess 2023-08-16 14:31:31 -04:00
parent 7f7c95b771
commit 9286769d2c
No known key found for this signature in database
GPG key ID: DB12DB0FF05F8F38
28 changed files with 86 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
Assistant/DaemonStatus.hs
View file

@ -13,6 +13,7 @@ import Assistant.Common
import Assistant.Alert.Utility
import Utility.Tmp
import Utility.NotificationBroadcaster
import Types.Availability
import Types.Transfer
import Logs.Transfer
import Logs.Trust
@ -59,6 +60,7 @@ calcSyncRemotes = do
let (exportremotes, nonexportremotes) = partition (exportTree . Remote.config) contentremotes
let isimport r = importTree (Remote.config r) || Remote.thirdPartyPopulated (Remote.remotetype r)
let dataremotes = filter (not . isimport) nonexportremotes
tocloud <- anyM iscloud contentremotes
return $ \dstatus -> dstatus
{ syncRemotes = syncable
@ -66,10 +68,14 @@ calcSyncRemotes = do
, syncDataRemotes = dataremotes
, exportRemotes = exportremotes
, downloadRemotes = contentremotes
, syncingToCloudRemote = any iscloud contentremotes
, syncingToCloudRemote = tocloud
}
where
iscloud r = not (Remote.readonly r) && Remote.availability r == Remote.GloballyAvailable
iscloud r
| Remote.readonly r = pure False
| otherwise = tryNonAsync (Remote.availability r) >>= return . \case
Right GloballyAvailable -> True
_ -> False
{- Updates the syncRemotes list from the list of all remotes in Annex state. -}
updateSyncRemotes :: Assistant ()

5
CHANGELOG
View file

@ -8,6 +8,11 @@ git-annex (10.20230803) UNRELEASED; urgency=medium
directory does not exist. An empty tree was imported, rather than the
import failing.
* Stop bundling curl in the OSX dmg and linux standalone image.
* sync, assist, push, pull: Skip more types of remotes when they
are not present due to eg being on a drive that is offline.
* Added AVAILABILITY UNAVAILABLE and the UNAVAILABLERESPONSE extension
to the external special remote protocol.
* The remote.name.annex-availability git config is no longer used.
-- Joey Hess <id@joeyh.name> Mon, 07 Aug 2023 13:04:13 -0400

8
Command/Sync.hs
View file

@ -78,6 +78,7 @@ import Annex.Import
import Annex.CheckIgnore
import Types.FileMatcher
import Types.GitConfig
import Types.Availability
import qualified Database.Export as Export
import Utility.Bloom
import Utility.OptParse
@ -388,10 +389,9 @@ syncRemotes' ps available =
listed = concat <$> mapM Remote.byNameOrGroup ps
good r
| Remote.gitSyncableRemoteType (Remote.remotetype r) =
Remote.Git.repoAvail =<< Remote.getRepo r
| otherwise = return True
good r = tryNonAsync (Remote.availability r) >>= return . \case
Right Unavailable -> False
_ -> True
fastest = fromMaybe [] . headMaybe . Remote.byCost

2
Remote/Adb.hs
View file

@ -113,7 +113,7 @@ gen r u rc gc rs = do
, gitconfig = gc
, localpath = Nothing
, remotetype = remote
, availability = LocallyAvailable
, availability = pure LocallyAvailable
, readonly = False
, appendonly = False
, untrustworthy = False

2
Remote/BitTorrent.hs
View file

@ -88,7 +88,7 @@ gen r _ rc gc rs = do
, readonly = True
, appendonly = False
, untrustworthy = False
, availability = GloballyAvailable
, availability = pure GloballyAvailable
, remotetype = remote
, mkUnavailable = return Nothing
, getInfo = return []

3
Remote/Borg.hs
View file

@ -112,7 +112,8 @@ gen r u rc gc rs = do
, gitconfig = gc
, localpath = borgRepoLocalPath borgrepo
, remotetype = remote
, availability = if borgLocal borgrepo then LocallyAvailable else GloballyAvailable
, availability = pure $
if borgLocal borgrepo then LocallyAvailable else GloballyAvailable
, readonly = False
, appendonly = False
-- When the user sets the appendonly field, they are

3
Remote/Bup.hs
View file

@ -97,7 +97,8 @@ gen r u rc gc rs = do
then Just buprepo
else Nothing
, remotetype = remote
, availability = if bupLocal buprepo then LocallyAvailable else GloballyAvailable
, availability = pure $
if bupLocal buprepo then LocallyAvailable else GloballyAvailable
, readonly = False
, appendonly = False
, untrustworthy = False

3
Remote/Ddar.hs
View file

@ -98,7 +98,8 @@ gen r u rc gc rs = do
then Just $ ddarRepoLocation ddarrepo
else Nothing
, remotetype = remote
, availability = if ddarLocal ddarrepo then LocallyAvailable else GloballyAvailable
, availability = pure $
if ddarLocal ddarrepo then LocallyAvailable else GloballyAvailable
, readonly = False
, appendonly = False
, untrustworthy = False

2
Remote/Directory.hs
View file

@ -134,7 +134,7 @@ gen r u rc gc rs = do
, readonly = False
, appendonly = False
, untrustworthy = False
, availability = LocallyAvailable
, availability = pure LocallyAvailable
, remotetype = remote
, mkUnavailable = gen r u rc
(gc { remoteAnnexDirectory = Just "/dev/null" }) rs

28
Remote/External.hs
View file

@ -68,7 +68,7 @@ gen r u rc gc rs
| externaltype == "readonly" = do
c <- parsedRemoteConfig remote rc
cst <- remoteCost gc c expensiveRemoteCost
let rmt = mk c cst GloballyAvailable
let rmt = mk c cst (pure GloballyAvailable)
Nothing
(externalInfo externaltype)
Nothing
@ -87,7 +87,6 @@ gen r u rc gc rs
(Git.remoteName r) (Just rs)
Annex.addCleanupAction (RemoteCleanup u) $ stopExternal external
cst <- getCost external r gc c
avail <- getAvailability external r gc
exportsupported <- if exportTree c
then checkExportSupported' external
else return False
@ -107,7 +106,7 @@ gen r u rc gc rs
let cheapexportsupported = if exportsupported
then exportIsSupported
else exportUnsupported
let rmt = mk c cst avail
let rmt = mk c cst (getAvailability external)
(Just (whereisKeyM external))
(getInfoM external)
(Just (claimUrlM external))
@ -777,25 +776,16 @@ getCost external r gc pc =
return c
defcst = expensiveRemoteCost
{- Caches the availability in the git config to avoid needing to start up an
- external special remote every time time just to ask it what its
- availability is.
-
- Most remotes do not bother to implement a reply to this request;
{- Most remotes do not bother to implement a reply to this request;
- globally available is the default.
-}
getAvailability :: External -> Git.Repo -> RemoteGitConfig -> Annex Availability
getAvailability external r gc =
maybe (catchNonAsync query (const (pure defavail))) return
(remoteAnnexAvailability gc)
getAvailability :: External -> Annex Availability
getAvailability external = catchNonAsync query (const (pure defavail))
where
query = do
avail <- handleRequest external GETAVAILABILITY Nothing $ \req -> case req of
AVAILABILITY avail -> result avail
UNSUPPORTED_REQUEST -> result defavail
_ -> Nothing
setRemoteAvailability r avail
return avail
query = handleRequest external GETAVAILABILITY Nothing $ \req -> case req of
AVAILABILITY avail -> result avail
UNSUPPORTED_REQUEST -> result defavail
_ -> Nothing
defavail = GloballyAvailable
claimUrlM :: External -> URLString -> Annex Bool

3
Remote/External/Types.hs vendored
View file

@ -110,6 +110,7 @@ supportedExtensionList :: ExtensionList
supportedExtensionList = ExtensionList
[ "INFO"
, "GETGITREMOTENAME"
, "UNAVAILABLERESPONSE"
, asyncExtension
]
@ -447,9 +448,11 @@ instance Proto.Serializable Size where
instance Proto.Serializable Availability where
serialize GloballyAvailable = "GLOBAL"
serialize LocallyAvailable = "LOCAL"
serialize Unavailable = "UNAVAILABLE"
deserialize "GLOBAL" = Just GloballyAvailable
deserialize "LOCAL" = Just LocallyAvailable
deserialize "UNAVAILABLE" = Just Unavailable
deserialize _ = Nothing
instance Proto.Serializable [(URLString, Size, FilePath)] where

2
Remote/GCrypt.hs
View file

@ -158,7 +158,7 @@ gen' r u c gc rs = do
, readonly = Git.repoIsHttp r
, appendonly = False
, untrustworthy = False
, availability = availabilityCalc r
, availability = pure (availabilityCalc r)
, remotetype = remote
, mkUnavailable = return Nothing
, getInfo = gitRepoInfo this

25
Remote/Git.hs
View file

@ -1,6 +1,6 @@
{- Standard git remotes.
-
- Copyright 2011-2021 Joey Hess <id@joeyh.name>
- Copyright 2011-2023 Joey Hess <id@joeyh.name>
-
- Licensed under the GNU AGPL version 3 or higher.
-}
@ -11,7 +11,6 @@
module Remote.Git (
remote,
configRead,
repoAvail,
onLocalRepo,
) where
@ -210,7 +209,7 @@ gen r u rc gc rs
, readonly = Git.repoIsHttp r
, appendonly = False
, untrustworthy = False
, availability = availabilityCalc r
, availability = repoAvail r
, remotetype = remote
, mkUnavailable = unavailable r u rc gc rs
, getInfo = gitRepoInfo new
@ -233,20 +232,24 @@ unavailable r = gen r'
_ -> r -- already unavailable
{- Checks relatively inexpensively if a repository is available for use. -}
repoAvail :: Git.Repo -> Annex Bool
repoAvail :: Git.Repo -> Annex Availability
repoAvail r
| Git.repoIsHttp r = return True
| Git.repoIsHttp r = return GloballyAvailable
| Git.GCrypt.isEncrypted r = do
g <- gitRepo
liftIO $ do
er <- Git.GCrypt.encryptedRemote g r
if Git.repoIsLocal er || Git.repoIsLocalUnknown er
then catchBoolIO $
void (Git.Config.read er) >> return True
else return True
| Git.repoIsUrl r = return True
| Git.repoIsLocalUnknown r = return False
| otherwise = liftIO $ isJust <$> catchMaybeIO (Git.Config.read r)
then checklocal er
else return GloballyAvailable
| Git.repoIsUrl r = return GloballyAvailable
| Git.repoIsLocalUnknown r = return Unavailable
| otherwise = checklocal r
where
checklocal r' = ifM (liftIO $ isJust <$> catchMaybeIO (Git.Config.read r'))
( return LocallyAvailable
, return Unavailable
)
{- Tries to read the config for a specified remote, updates state, and
- returns the updated repo. -}

2
Remote/GitLFS.hs
View file

@ -130,7 +130,7 @@ gen r u rc gc rs = do
, gitconfig = gc
, localpath = Nothing
, remotetype = remote
, availability = GloballyAvailable
, availability = pure GloballyAvailable
, readonly = False
-- content cannot be removed from a git-lfs repo
, appendonly = True

2
Remote/Glacier.hs
View file

@ -103,7 +103,7 @@ gen r u rc gc rs = do
, readonly = False
, appendonly = False
, untrustworthy = False
, availability = GloballyAvailable
, availability = pure GloballyAvailable
, remotetype = remote
, mkUnavailable = return Nothing
, getInfo = includeCredsInfo c (AWS.creds u) $

2
Remote/Hook.hs
View file

@ -82,7 +82,7 @@ gen r u rc gc rs = do
, readonly = False
, appendonly = False
, untrustworthy = False
, availability = GloballyAvailable
, availability = pure GloballyAvailable
, remotetype = remote
, mkUnavailable = gen r u rc
(gc { remoteAnnexHookType = Just "!dne!" })

2
Remote/HttpAlso.hs
View file

@ -95,7 +95,7 @@ gen r u rc gc rs = do
, readonly = True
, appendonly = False
, untrustworthy = False
, availability = GloballyAvailable
, availability = pure GloballyAvailable
, remotetype = remote
, mkUnavailable = return Nothing
, getInfo = return []

2
Remote/P2P.hs
View file

@ -77,7 +77,7 @@ chainGen addr r u rc gc rs = do
, readonly = False
, appendonly = False
, untrustworthy = False
, availability = GloballyAvailable
, availability = pure GloballyAvailable
, remotetype = remote
, mkUnavailable = return Nothing
, getInfo = gitRepoInfo this

3
Remote/Rsync.hs
View file

@ -120,7 +120,8 @@ gen r u rc gc rs = do
, readonly = False
, appendonly = False
, untrustworthy = False
, availability = if islocal then LocallyAvailable else GloballyAvailable
, availability = pure $
if islocal then LocallyAvailable else GloballyAvailable
, remotetype = remote
, mkUnavailable = return Nothing
, getInfo = return [("url", url)]

2
Remote/S3.hs
View file

@ -249,7 +249,7 @@ gen r u rc gc rs = do
, readonly = False
, appendonly = False
, untrustworthy = False
, availability = GloballyAvailable
, availability = pure GloballyAvailable
, remotetype = remote
, mkUnavailable = gen r u (M.insert hostField (Proposed "!dne!") rc) gc rs
, getInfo = includeCredsInfo c (AWS.creds u) (s3Info c info)

2
Remote/Tahoe.hs
View file

@ -108,7 +108,7 @@ gen r u rc gc rs = do
, readonly = False
, appendonly = False
, untrustworthy = False
, availability = GloballyAvailable
, availability = pure GloballyAvailable
, remotetype = remote
, mkUnavailable = return Nothing
, getInfo = return []

2
Remote/Web.hs
View file

@ -93,7 +93,7 @@ gen r u rc gc rs = do
, readonly = True
, appendonly = False
, untrustworthy = False
, availability = GloballyAvailable
, availability = pure GloballyAvailable
, remotetype = remote
, mkUnavailable = return Nothing
, getInfo = return []

2
Remote/WebDAV.hs
View file

@ -117,7 +117,7 @@ gen r u rc gc rs = do
, readonly = False
, appendonly = False
, untrustworthy = False
, availability = GloballyAvailable
, availability = pure GloballyAvailable
, remotetype = remote
, mkUnavailable = gen r u (M.insert urlField (Proposed "http://!dne!/") rc) gc rs
, getInfo = includeCredsInfo c (davCreds u) $

4
Types/Availability.hs
View file

@ -7,5 +7,5 @@
module Types.Availability where
data Availability = GloballyAvailable | LocallyAvailable
deriving (Eq, Show, Read)
data Availability = GloballyAvailable | LocallyAvailable | Unavailable
deriving (Eq, Show)

3
Types/GitConfig.hs
View file

@ -37,7 +37,6 @@ import Utility.DataUnits
import Config.Cost
import Types.UUID
import Types.Distribution
import Types.Availability
import Types.Concurrency
import Types.NumCopies
import Types.Difference
@ -351,7 +350,6 @@ data RemoteGitConfig = RemoteGitConfig
, remoteAnnexTrustLevel :: Maybe String
, remoteAnnexStartCommand :: Maybe String
, remoteAnnexStopCommand :: Maybe String
, remoteAnnexAvailability :: Maybe Availability
, remoteAnnexSpeculatePresent :: Bool
, remoteAnnexBare :: Maybe Bool
, remoteAnnexRetry :: Maybe Integer
@ -416,7 +414,6 @@ extractRemoteGitConfig r remotename = do
, remoteAnnexTrustLevel = notempty $ getmaybe "trustlevel"
, remoteAnnexStartCommand = notempty $ getmaybe "start-command"
, remoteAnnexStopCommand = notempty $ getmaybe "stop-command"
, remoteAnnexAvailability = getmayberead "availability"
, remoteAnnexSpeculatePresent = getbool "speculate-present" False
, remoteAnnexBare = getmaybebool "bare"
, remoteAnnexRetry = getmayberead "retry"

3
Types/Remote.hs
View file

@ -152,7 +152,8 @@ data RemoteA a = Remote
-- decide.
, untrustworthy :: Bool
-- a Remote can be globally available. (Ie, "in the cloud".)
, availability :: Availability
-- Some Remotes can mark themselves unavailable.
, availability :: a Availability
-- the type of the remote
, remotetype :: RemoteTypeA a
-- For testing, makes a version of this remote that is not

21
doc/design/external_special_remote_protocol.mdwn
View file

@ -45,7 +45,7 @@ Recent versions of git-annex respond with a message indicating
protocol extensions that it supports. Older versions of
git-annex do not send this message.
EXTENSIONS INFO ASYNC GETGITREMOTENAME
EXTENSIONS INFO ASYNC GETGITREMOTENAME UNAVAILABLERESPONSE
The special remote can respond to that with its own EXTENSIONS message, listing
any extensions it wants to use.
@ -207,9 +207,19 @@ the special remote can reply with `UNSUPPORTED-REQUEST`.
(Ie stored in the cloud vs on a local disk.)
If the remote replies with `UNSUPPORTED-REQUEST`, its availability
is assumed to be global. So, only remotes that are only reachable
locally need to worry about implementing this.
locally need to worry about implementing this.
This is queried at remote startup, so should avoid doing anything that
can take long to run or is expensive. Checking if a directory where the
remote stores files is currently mounted is the kind of thing it makes
sense to do here.
* `AVAILABILITY GLOBAL|LOCAL`
Indicates if the remote is globally or only locally available.
* `AVAILABILITY UNAVAILABLE`
Indicates that the remote is not currently available.
This will prevent some git-annex commands like `git-annex sync` from
trying to use the remote.
Older versions of git-annex do not support this response, so avoid
sending it unless the `UNAVAILABLERESPONSE` extension is enabled.
* `CLAIMURL Url`
Asks the remote if it wishes to claim responsibility for downloading
an url.
@ -445,13 +455,16 @@ avoid talking to the buggy old version of git-annex.
These protocol extensions are currently supported.
* `INFO`
This makes the `INFO` message available to use.
This allows using the `INFO` message.
* `ASYNC`
This lets multiple actions be performed at the same time by
a single external special remote program, rather than starting multiple
programs. See the [[async_appendix]] for details.
* `GETGITREMOTENAME`
This makes the `GETGITREMOTENAME` message available to use.
This allows using the `GETGITREMOTENAME` message.
* `UNAVAILABLERESPONSE`
This allows the `AVAILABILITY UNAVAILABLE` response to be used
in reply to `GETAVAILABILITY`.
## signals

3
doc/git-annex.mdwn
View file

@ -1565,8 +1565,7 @@ Remotes are configured using these settings in `.git/config`.
* `remote.<name>.annex-availability`
Can be used to tell git-annex whether a remote is LocallyAvailable
or GloballyAvailable. Normally, git-annex determines this automatically.
This configuration setting is no longer used.
* `remote.<name>.annex-speculate-present`