2011-06-02 01:56:04 +00:00
|
|
|
{- git-annex remotes types
|
2011-03-27 21:12:32 +00:00
|
|
|
-
|
2011-12-31 08:14:33 +00:00
|
|
|
- Most things should not need this, using Types instead
|
2011-03-27 19:56:43 +00:00
|
|
|
-
|
2015-01-21 16:50:09 +00:00
|
|
|
- Copyright 2011-2014 Joey Hess <id@joeyh.name>
|
2011-03-27 19:56:43 +00:00
|
|
|
-
|
|
|
|
- Licensed under the GNU GPL version 3 or higher.
|
|
|
|
-}
|
|
|
|
|
2015-10-08 19:01:38 +00:00
|
|
|
{-# LANGUAGE RankNTypes #-}
|
|
|
|
|
2014-01-13 18:41:10 +00:00
|
|
|
module Types.Remote
|
|
|
|
( RemoteConfigKey
|
|
|
|
, RemoteConfig
|
|
|
|
, RemoteTypeA(..)
|
|
|
|
, RemoteA(..)
|
2017-02-07 18:35:58 +00:00
|
|
|
, SetupStage(..)
|
2014-01-13 18:41:10 +00:00
|
|
|
, Availability(..)
|
other 80% of avoding verification when hard linking to objects in shared repo
In c6632ee5c8e66c26ef18317f56ae02bae1e7e280, it actually only handled
uploading objects to a shared repository. To avoid verification when
downloading objects from a shared repository, was a lot harder.
On the plus side, if the process of downloading a file from a remote
is able to verify its content on the side, the remote can indicate this
now, and avoid the extra post-download verification.
As of yet, I don't have any remotes (except Git) using this ability.
Some more work would be needed to support it in special remotes.
It would make sense for tahoe to implicitly verify things downloaded from it;
as long as you trust your tahoe server (which typically runs locally),
there's cryptographic integrity. OTOH, despite bup being based on shas,
a bup repo under an attacker's control could have the git ref used for an
object changed, and so a bup repo shouldn't implicitly verify. Indeed,
tahoe seems unique in being trustworthy enough to implicitly verify.
2015-10-02 17:56:42 +00:00
|
|
|
, Verification(..)
|
|
|
|
, unVerified
|
2014-01-13 18:41:10 +00:00
|
|
|
)
|
|
|
|
where
|
2011-03-27 19:56:43 +00:00
|
|
|
|
2011-03-29 03:51:07 +00:00
|
|
|
import Data.Map as M
|
2011-07-15 07:12:05 +00:00
|
|
|
import Data.Ord
|
2011-03-27 19:56:43 +00:00
|
|
|
|
2011-06-30 17:16:57 +00:00
|
|
|
import qualified Git
|
2011-06-02 01:56:04 +00:00
|
|
|
import Types.Key
|
2011-11-07 18:46:01 +00:00
|
|
|
import Types.UUID
|
2013-01-01 17:52:47 +00:00
|
|
|
import Types.GitConfig
|
2014-01-13 18:41:10 +00:00
|
|
|
import Types.Availability
|
2014-02-11 18:06:50 +00:00
|
|
|
import Types.Creds
|
2014-12-11 19:32:42 +00:00
|
|
|
import Types.UrlContents
|
2015-10-09 16:36:04 +00:00
|
|
|
import Types.NumCopies
|
2013-03-13 20:16:01 +00:00
|
|
|
import Config.Cost
|
2013-03-28 21:03:04 +00:00
|
|
|
import Utility.Metered
|
2013-11-07 22:02:00 +00:00
|
|
|
import Git.Types
|
2013-10-11 20:03:18 +00:00
|
|
|
import Utility.SafeCommand
|
2014-12-08 17:40:15 +00:00
|
|
|
import Utility.Url
|
2011-03-27 19:56:43 +00:00
|
|
|
|
2012-11-14 23:32:27 +00:00
|
|
|
type RemoteConfigKey = String
|
2017-02-07 18:35:58 +00:00
|
|
|
|
2012-11-14 23:32:27 +00:00
|
|
|
type RemoteConfig = M.Map RemoteConfigKey String
|
2011-04-15 19:09:36 +00:00
|
|
|
|
2017-02-07 18:35:58 +00:00
|
|
|
data SetupStage = Init | Enable
|
|
|
|
deriving (Eq)
|
|
|
|
|
2011-03-29 03:51:07 +00:00
|
|
|
{- There are different types of remotes. -}
|
2011-12-31 08:11:39 +00:00
|
|
|
data RemoteTypeA a = RemoteType {
|
2011-03-29 03:51:07 +00:00
|
|
|
-- human visible type name
|
|
|
|
typename :: String,
|
2011-03-29 21:57:20 +00:00
|
|
|
-- enumerates remotes of this type
|
2015-08-05 17:49:54 +00:00
|
|
|
-- The Bool is True if automatic initialization of remotes is desired
|
|
|
|
enumerate :: Bool -> a [Git.Repo],
|
2011-03-29 21:57:20 +00:00
|
|
|
-- generates a remote of this type
|
2013-09-12 19:54:35 +00:00
|
|
|
generate :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> a (Maybe (RemoteA a)),
|
2017-02-07 18:35:58 +00:00
|
|
|
-- initializes or enables a remote
|
|
|
|
setup :: SetupStage -> Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> a (RemoteConfig, UUID)
|
2011-03-29 03:51:07 +00:00
|
|
|
}
|
|
|
|
|
2011-12-31 08:11:39 +00:00
|
|
|
instance Eq (RemoteTypeA a) where
|
2011-12-31 07:27:37 +00:00
|
|
|
x == y = typename x == typename y
|
|
|
|
|
2011-03-29 03:51:07 +00:00
|
|
|
{- An individual remote. -}
|
2011-12-31 08:11:39 +00:00
|
|
|
data RemoteA a = Remote {
|
2011-03-27 19:56:43 +00:00
|
|
|
-- each Remote has a unique uuid
|
2011-11-07 18:46:01 +00:00
|
|
|
uuid :: UUID,
|
2011-03-27 19:56:43 +00:00
|
|
|
-- each Remote has a human visible name
|
2013-09-27 03:28:25 +00:00
|
|
|
name :: RemoteName,
|
2011-03-27 19:56:43 +00:00
|
|
|
-- Remotes have a use cost; higher is more expensive
|
2013-03-13 20:16:01 +00:00
|
|
|
cost :: Cost,
|
2014-07-26 17:25:06 +00:00
|
|
|
-- Transfers a key's contents from disk to the remote.
|
resume interrupted chunked uploads
Leverage the new chunked remotes to automatically resume uploads.
Sort of like rsync, although of course not as efficient since this
needs to start at a chunk boundry.
But, unlike rsync, this method will work for S3, WebDAV, external
special remotes, etc, etc. Only directory special remotes so far,
but many more soon!
This implementation will also allow starting an upload from one repository,
interrupting it, and then resuming the upload to the same remote from
an entirely different repository.
Note that I added a comment that storeKey should atomically move the content
into place once it's all received. This was already an undocumented
requirement -- it's necessary for hasKey to work reliably. This resume code
just uses hasKey to find the first chunk that's missing.
Note that if there are two uploads of the same key to the same chunked remote,
one might resume at the point the other had gotten to, but both will then
redundantly upload. As before.
In the non-resume case, this adds one hasKey call per storeKey, and only
if the remote is configured to use chunks. Future work: Try to eliminate that
hasKey. Notice that eg, `git annex copy --to` checks if the key is present
before sending it, so is already running hasKey.. which could perhaps
be cached and reused.
However, this additional overhead is not very large compared with
transferring an entire large file, and the ability to resume
is certianly worth it. There is an optimisation in place for small files,
that avoids trying to resume if the whole file fits within one chunk.
This commit was sponsored by Georg Bauer.
2014-07-28 18:18:08 +00:00
|
|
|
-- The key should not appear to be present on the remote until
|
|
|
|
-- all of its contents have been transferred.
|
2012-09-21 18:50:14 +00:00
|
|
|
storeKey :: Key -> AssociatedFile -> MeterUpdate -> a Bool,
|
2013-04-11 21:15:45 +00:00
|
|
|
-- Retrieves a key's contents to a file.
|
other 80% of avoding verification when hard linking to objects in shared repo
In c6632ee5c8e66c26ef18317f56ae02bae1e7e280, it actually only handled
uploading objects to a shared repository. To avoid verification when
downloading objects from a shared repository, was a lot harder.
On the plus side, if the process of downloading a file from a remote
is able to verify its content on the side, the remote can indicate this
now, and avoid the extra post-download verification.
As of yet, I don't have any remotes (except Git) using this ability.
Some more work would be needed to support it in special remotes.
It would make sense for tahoe to implicitly verify things downloaded from it;
as long as you trust your tahoe server (which typically runs locally),
there's cryptographic integrity. OTOH, despite bup being based on shas,
a bup repo under an attacker's control could have the git ref used for an
object changed, and so a bup repo shouldn't implicitly verify. Indeed,
tahoe seems unique in being trustworthy enough to implicitly verify.
2015-10-02 17:56:42 +00:00
|
|
|
-- (The MeterUpdate does not need to be used if it writes
|
|
|
|
-- sequentially to the file.)
|
|
|
|
retrieveKeyFile :: Key -> AssociatedFile -> FilePath -> MeterUpdate -> a (Bool, Verification),
|
2015-04-18 17:07:57 +00:00
|
|
|
-- Retrieves a key's contents to a tmp file, if it can be done cheaply.
|
|
|
|
-- It's ok to create a symlink or hardlink.
|
2015-04-14 20:35:10 +00:00
|
|
|
retrieveKeyFileCheap :: Key -> AssociatedFile -> FilePath -> a Bool,
|
2015-10-08 19:01:38 +00:00
|
|
|
-- Removes a key's contents (succeeds if the contents are not present)
|
2011-03-27 20:17:56 +00:00
|
|
|
removeKey :: Key -> a Bool,
|
2015-10-08 19:01:38 +00:00
|
|
|
-- Uses locking to prevent removal of a key's contents,
|
2015-10-09 17:07:03 +00:00
|
|
|
-- thus producing a VerifiedCopy, which is passed to the callback.
|
|
|
|
-- If unable to lock, does not run the callback, and throws an
|
|
|
|
-- error.
|
2015-10-08 19:01:38 +00:00
|
|
|
-- This is optional; remotes do not have to support locking.
|
2015-10-09 17:07:03 +00:00
|
|
|
lockContent :: forall r. Maybe (Key -> (VerifiedCopy -> a r) -> a r),
|
2014-08-06 17:45:19 +00:00
|
|
|
-- Checks if a key is present in the remote.
|
|
|
|
-- Throws an exception if the remote cannot be accessed.
|
|
|
|
checkPresent :: Key -> a Bool,
|
|
|
|
-- Some remotes can checkPresent without an expensive network
|
2011-03-27 19:56:43 +00:00
|
|
|
-- operation.
|
2014-08-06 17:45:19 +00:00
|
|
|
checkPresentCheap :: Bool,
|
2012-02-14 07:49:48 +00:00
|
|
|
-- Some remotes can provide additional details for whereis.
|
|
|
|
whereisKey :: Maybe (Key -> a [String]),
|
2013-10-11 20:03:18 +00:00
|
|
|
-- Some remotes can run a fsck operation on the remote,
|
|
|
|
-- without transferring all the data to the local repo
|
|
|
|
-- The parameters are passed to the fsck command on the remote.
|
|
|
|
remoteFsck :: Maybe ([CommandParam] -> a (IO Bool)),
|
2013-10-27 19:38:59 +00:00
|
|
|
-- Runs an action to repair the remote's git repository.
|
|
|
|
repairRepo :: Maybe (a Bool -> a (IO Bool)),
|
2012-11-30 04:55:59 +00:00
|
|
|
-- a Remote has a persistent configuration store
|
|
|
|
config :: RemoteConfig,
|
2013-01-01 17:52:47 +00:00
|
|
|
-- git repo for the Remote
|
2011-12-31 07:27:37 +00:00
|
|
|
repo :: Git.Repo,
|
2013-01-01 17:52:47 +00:00
|
|
|
-- a Remote's configuration from git
|
|
|
|
gitconfig :: RemoteGitConfig,
|
2012-08-26 18:26:43 +00:00
|
|
|
-- a Remote can be assocated with a specific local filesystem path
|
|
|
|
localpath :: Maybe FilePath,
|
2012-08-26 19:39:02 +00:00
|
|
|
-- a Remote can be known to be readonly
|
|
|
|
readonly :: Bool,
|
2013-03-15 23:16:13 +00:00
|
|
|
-- a Remote can be globally available. (Ie, "in the cloud".)
|
2014-01-13 18:41:10 +00:00
|
|
|
availability :: Availability,
|
2011-12-31 07:27:37 +00:00
|
|
|
-- the type of the remote
|
2014-08-10 18:52:58 +00:00
|
|
|
remotetype :: RemoteTypeA a,
|
|
|
|
-- For testing, makes a version of this remote that is not
|
|
|
|
-- available for use. All its actions should fail.
|
2014-10-21 18:36:09 +00:00
|
|
|
mkUnavailable :: a (Maybe (RemoteA a)),
|
|
|
|
-- Information about the remote, for git annex info to display.
|
2014-12-08 17:40:15 +00:00
|
|
|
getInfo :: a [(String, String)],
|
|
|
|
-- Some remotes can download from an url (or uri).
|
2014-12-11 18:09:57 +00:00
|
|
|
claimUrl :: Maybe (URLString -> a Bool),
|
2014-12-11 19:32:42 +00:00
|
|
|
-- Checks that the url is accessible, and gets information about
|
|
|
|
-- its contents, without downloading the full content.
|
2014-12-08 23:14:24 +00:00
|
|
|
-- Throws an exception if the url is inaccessible.
|
2014-12-11 19:32:42 +00:00
|
|
|
checkUrl :: Maybe (URLString -> a UrlContents)
|
2011-03-27 19:56:43 +00:00
|
|
|
}
|
|
|
|
|
2011-12-31 08:11:39 +00:00
|
|
|
instance Show (RemoteA a) where
|
2011-03-30 19:15:46 +00:00
|
|
|
show remote = "Remote { name =\"" ++ name remote ++ "\" }"
|
2011-03-27 19:56:43 +00:00
|
|
|
|
|
|
|
-- two remotes are the same if they have the same uuid
|
2011-12-31 08:11:39 +00:00
|
|
|
instance Eq (RemoteA a) where
|
2011-03-27 20:17:56 +00:00
|
|
|
x == y = uuid x == uuid y
|
2011-03-27 19:56:43 +00:00
|
|
|
|
2011-12-31 08:11:39 +00:00
|
|
|
instance Ord (RemoteA a) where
|
2013-03-16 21:43:42 +00:00
|
|
|
compare = comparing uuid
|
other 80% of avoding verification when hard linking to objects in shared repo
In c6632ee5c8e66c26ef18317f56ae02bae1e7e280, it actually only handled
uploading objects to a shared repository. To avoid verification when
downloading objects from a shared repository, was a lot harder.
On the plus side, if the process of downloading a file from a remote
is able to verify its content on the side, the remote can indicate this
now, and avoid the extra post-download verification.
As of yet, I don't have any remotes (except Git) using this ability.
Some more work would be needed to support it in special remotes.
It would make sense for tahoe to implicitly verify things downloaded from it;
as long as you trust your tahoe server (which typically runs locally),
there's cryptographic integrity. OTOH, despite bup being based on shas,
a bup repo under an attacker's control could have the git ref used for an
object changed, and so a bup repo shouldn't implicitly verify. Indeed,
tahoe seems unique in being trustworthy enough to implicitly verify.
2015-10-02 17:56:42 +00:00
|
|
|
|
2015-10-08 21:58:32 +00:00
|
|
|
instance ToUUID (RemoteA a) where
|
|
|
|
toUUID = uuid
|
|
|
|
|
other 80% of avoding verification when hard linking to objects in shared repo
In c6632ee5c8e66c26ef18317f56ae02bae1e7e280, it actually only handled
uploading objects to a shared repository. To avoid verification when
downloading objects from a shared repository, was a lot harder.
On the plus side, if the process of downloading a file from a remote
is able to verify its content on the side, the remote can indicate this
now, and avoid the extra post-download verification.
As of yet, I don't have any remotes (except Git) using this ability.
Some more work would be needed to support it in special remotes.
It would make sense for tahoe to implicitly verify things downloaded from it;
as long as you trust your tahoe server (which typically runs locally),
there's cryptographic integrity. OTOH, despite bup being based on shas,
a bup repo under an attacker's control could have the git ref used for an
object changed, and so a bup repo shouldn't implicitly verify. Indeed,
tahoe seems unique in being trustworthy enough to implicitly verify.
2015-10-02 17:56:42 +00:00
|
|
|
-- Use Verified when the content of a key is verified as part of a
|
|
|
|
-- transfer, and so a separate verification step is not needed.
|
|
|
|
data Verification = UnVerified | Verified
|
|
|
|
|
|
|
|
unVerified :: Monad m => m Bool -> m (Bool, Verification)
|
|
|
|
unVerified a = do
|
|
|
|
ok <- a
|
|
|
|
return (ok, UnVerified)
|