2018-03-08 16:11:00 -04:00
|
|
|
{- Helpers for remotes using the git-annex P2P protocol.
|
|
|
|
-
|
2021-02-09 17:03:27 -04:00
|
|
|
- Copyright 2016-2021 Joey Hess <id@joeyh.name>
|
2018-03-08 16:11:00 -04:00
|
|
|
-
|
2019-03-13 15:48:14 -04:00
|
|
|
- Licensed under the GNU AGPL version 3 or higher.
|
2018-03-08 16:11:00 -04:00
|
|
|
-}
|
|
|
|
|
|
|
|
{-# LANGUAGE RankNTypes #-}
|
|
|
|
|
|
|
|
module Remote.Helper.P2P where
|
|
|
|
|
|
|
|
import Annex.Common
|
|
|
|
import qualified P2P.Protocol as P2P
|
|
|
|
import P2P.IO
|
|
|
|
import Types.Remote
|
|
|
|
import Annex.Content
|
|
|
|
import Messages.Progress
|
|
|
|
import Utility.Metered
|
|
|
|
import Types.NumCopies
|
2021-07-27 14:07:23 -04:00
|
|
|
import Annex.Verify
|
2018-03-08 16:11:00 -04:00
|
|
|
|
|
|
|
import Control.Concurrent
|
|
|
|
|
|
|
|
-- Runs a Proto action using a connection it sets up.
|
|
|
|
type ProtoRunner a = P2P.Proto a -> Annex (Maybe a)
|
|
|
|
|
|
|
|
-- Runs a Proto action using a ClosableConnection.
|
|
|
|
type ProtoConnRunner c = forall a. P2P.Proto a -> ClosableConnection c -> Annex (ClosableConnection c, Maybe a)
|
|
|
|
|
|
|
|
-- Runs an Annex action with a connection from the pool, adding it back to
|
|
|
|
-- the pool when done.
|
|
|
|
type WithConn a c = (ClosableConnection c -> Annex (ClosableConnection c, a)) -> Annex a
|
|
|
|
|
bwlimit
Added annex.bwlimit and remote.name.annex-bwlimit config that works for git
remotes and many but not all special remotes.
This nearly works, at least for a git remote on the same disk. With it set
to 100kb/1s, the meter displays an actual bandwidth of 128 kb/s, with
occasional spikes to 160 kb/s. So it needs to delay just a bit longer...
I'm unsure why.
However, at the beginning a lot of data flows before it determines the
right bandwidth limit. A granularity of less than 1s would probably improve
that.
And, I don't know yet if it makes sense to have it be 100ks/1s rather than
100kb/s. Is there a situation where the user would want a larger
granularity? Does granulatity need to be configurable at all? I only used that
format for the config really in order to reuse an existing parser.
This can't support for external special remotes, or for ones that
themselves shell out to an external command. (Well, it could, but it
would involve pausing and resuming the child process tree, which seems
very hard to implement and very strange besides.) There could also be some
built-in special remotes that it still doesn't work for, due to them not
having a progress meter whose displays blocks the bandwidth using thread.
But I don't think there are actually any that run a separate thread for
downloads than the thread that displays the progress meter.
Sponsored-by: Graham Spencer on Patreon
2021-09-21 16:58:02 -04:00
|
|
|
store :: RemoteGitConfig -> (MeterUpdate -> ProtoRunner Bool) -> Key -> AssociatedFile -> MeterUpdate -> Annex ()
|
|
|
|
store gc runner k af p = do
|
2020-11-05 11:26:34 -04:00
|
|
|
let sizer = KeySizer k (fmap (toRawFilePath . fst) <$> prepSendAnnex k)
|
2021-09-22 10:51:10 -04:00
|
|
|
let bwlimit = remoteAnnexBwLimit gc
|
bwlimit
Added annex.bwlimit and remote.name.annex-bwlimit config that works for git
remotes and many but not all special remotes.
This nearly works, at least for a git remote on the same disk. With it set
to 100kb/1s, the meter displays an actual bandwidth of 128 kb/s, with
occasional spikes to 160 kb/s. So it needs to delay just a bit longer...
I'm unsure why.
However, at the beginning a lot of data flows before it determines the
right bandwidth limit. A granularity of less than 1s would probably improve
that.
And, I don't know yet if it makes sense to have it be 100ks/1s rather than
100kb/s. Is there a situation where the user would want a larger
granularity? Does granulatity need to be configurable at all? I only used that
format for the config really in order to reuse an existing parser.
This can't support for external special remotes, or for ones that
themselves shell out to an external command. (Well, it could, but it
would involve pausing and resuming the child process tree, which seems
very hard to implement and very strange besides.) There could also be some
built-in special remotes that it still doesn't work for, due to them not
having a progress meter whose displays blocks the bandwidth using thread.
But I don't think there are actually any that run a separate thread for
downloads than the thread that displays the progress meter.
Sponsored-by: Graham Spencer on Patreon
2021-09-21 16:58:02 -04:00
|
|
|
metered (Just p) sizer bwlimit $ \_ p' ->
|
2020-05-13 14:03:00 -04:00
|
|
|
runner p' (P2P.put k af p') >>= \case
|
|
|
|
Just True -> return ()
|
2021-03-06 17:47:05 -04:00
|
|
|
Just False -> giveup "Transfer failed"
|
2020-05-14 14:08:09 -04:00
|
|
|
Nothing -> remoteUnavail
|
2018-03-08 16:11:00 -04:00
|
|
|
|
bwlimit
Added annex.bwlimit and remote.name.annex-bwlimit config that works for git
remotes and many but not all special remotes.
This nearly works, at least for a git remote on the same disk. With it set
to 100kb/1s, the meter displays an actual bandwidth of 128 kb/s, with
occasional spikes to 160 kb/s. So it needs to delay just a bit longer...
I'm unsure why.
However, at the beginning a lot of data flows before it determines the
right bandwidth limit. A granularity of less than 1s would probably improve
that.
And, I don't know yet if it makes sense to have it be 100ks/1s rather than
100kb/s. Is there a situation where the user would want a larger
granularity? Does granulatity need to be configurable at all? I only used that
format for the config really in order to reuse an existing parser.
This can't support for external special remotes, or for ones that
themselves shell out to an external command. (Well, it could, but it
would involve pausing and resuming the child process tree, which seems
very hard to implement and very strange besides.) There could also be some
built-in special remotes that it still doesn't work for, due to them not
having a progress meter whose displays blocks the bandwidth using thread.
But I don't think there are actually any that run a separate thread for
downloads than the thread that displays the progress meter.
Sponsored-by: Graham Spencer on Patreon
2021-09-21 16:58:02 -04:00
|
|
|
retrieve :: RemoteGitConfig -> (MeterUpdate -> ProtoRunner (Bool, Verification)) -> Key -> AssociatedFile -> FilePath -> MeterUpdate -> VerifyConfig -> Annex Verification
|
|
|
|
retrieve gc runner k af dest p verifyconfig = do
|
2021-02-09 17:03:27 -04:00
|
|
|
iv <- startVerifyKeyContentIncrementally verifyconfig k
|
2021-09-22 10:51:10 -04:00
|
|
|
let bwlimit = remoteAnnexBwLimit gc
|
bwlimit
Added annex.bwlimit and remote.name.annex-bwlimit config that works for git
remotes and many but not all special remotes.
This nearly works, at least for a git remote on the same disk. With it set
to 100kb/1s, the meter displays an actual bandwidth of 128 kb/s, with
occasional spikes to 160 kb/s. So it needs to delay just a bit longer...
I'm unsure why.
However, at the beginning a lot of data flows before it determines the
right bandwidth limit. A granularity of less than 1s would probably improve
that.
And, I don't know yet if it makes sense to have it be 100ks/1s rather than
100kb/s. Is there a situation where the user would want a larger
granularity? Does granulatity need to be configurable at all? I only used that
format for the config really in order to reuse an existing parser.
This can't support for external special remotes, or for ones that
themselves shell out to an external command. (Well, it could, but it
would involve pausing and resuming the child process tree, which seems
very hard to implement and very strange besides.) There could also be some
built-in special remotes that it still doesn't work for, due to them not
having a progress meter whose displays blocks the bandwidth using thread.
But I don't think there are actually any that run a separate thread for
downloads than the thread that displays the progress meter.
Sponsored-by: Graham Spencer on Patreon
2021-09-21 16:58:02 -04:00
|
|
|
metered (Just p) k bwlimit $ \m p' ->
|
2021-02-09 17:03:27 -04:00
|
|
|
runner p' (P2P.get dest k iv af m p') >>= \case
|
2020-05-13 17:05:56 -04:00
|
|
|
Just (True, v) -> return v
|
2021-03-06 17:47:05 -04:00
|
|
|
Just (False, _) -> giveup "Transfer failed"
|
2020-05-14 14:08:09 -04:00
|
|
|
Nothing -> remoteUnavail
|
2018-03-08 16:11:00 -04:00
|
|
|
|
2020-05-14 14:08:09 -04:00
|
|
|
remove :: ProtoRunner Bool -> Key -> Annex ()
|
|
|
|
remove runner k = runner (P2P.remove k) >>= \case
|
|
|
|
Just True -> return ()
|
|
|
|
Just False -> giveup "removing content from remote failed"
|
|
|
|
Nothing -> remoteUnavail
|
2018-03-08 16:11:00 -04:00
|
|
|
|
|
|
|
checkpresent :: ProtoRunner Bool -> Key -> Annex Bool
|
2020-05-14 14:08:09 -04:00
|
|
|
checkpresent runner k = maybe remoteUnavail return =<< runner (P2P.checkPresent k)
|
2018-03-08 16:11:00 -04:00
|
|
|
|
|
|
|
lock :: WithConn a c -> ProtoConnRunner c -> UUID -> Key -> (VerifiedCopy -> Annex a) -> Annex a
|
|
|
|
lock withconn connrunner u k callback = withconn $ \conn -> do
|
|
|
|
connv <- liftIO $ newMVar conn
|
|
|
|
let runproto d p = do
|
|
|
|
c <- liftIO $ takeMVar connv
|
|
|
|
(c', mr) <- connrunner p c
|
|
|
|
liftIO $ putMVar connv c'
|
|
|
|
return (fromMaybe d mr)
|
|
|
|
r <- P2P.lockContentWhile runproto k go
|
|
|
|
conn' <- liftIO $ takeMVar connv
|
|
|
|
return (conn', r)
|
|
|
|
where
|
|
|
|
go False = giveup "can't lock content"
|
|
|
|
go True = withVerifiedCopy LockedCopy u (return True) callback
|
2020-05-14 14:08:09 -04:00
|
|
|
|
|
|
|
remoteUnavail :: a
|
|
|
|
remoteUnavail = giveup "can't connect to remote"
|