From 3207e8293b4bd1aee8d1ea599a537bfb80290d1b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 18 Dec 2020 16:03:51 -0400 Subject: [PATCH] start borg special remote Compiles, but unusable so far. --- Remote/Borg.hs | 105 +++++++++++++++++++ Remote/List.hs | 2 + Types/GitConfig.hs | 2 + doc/devblog/day_637__thirdparty_of_borg.mdwn | 17 +++ doc/git-annex.mdwn | 6 ++ doc/special_remotes/borg.mdwn | 4 +- git-annex.cabal | 1 + 7 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 Remote/Borg.hs create mode 100644 doc/devblog/day_637__thirdparty_of_borg.mdwn diff --git a/Remote/Borg.hs b/Remote/Borg.hs new file mode 100644 index 0000000000..80e5be2964 --- /dev/null +++ b/Remote/Borg.hs @@ -0,0 +1,105 @@ +{- Using borg as a remote. + - + - Copyright 2020 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +module Remote.Borg (remote) where + +import Annex.Common +import Types.Remote +import Types.Creds +import qualified Git +import Config +import Config.Cost +import Annex.SpecialRemote.Config +import Remote.Helper.Special +import Remote.Helper.ExportImport +import Annex.UUID +import Types.ProposedAccepted + +import qualified Data.Map as M + +type BorgRepo = String + +remote :: RemoteType +remote = RemoteType + { typename = "borg" + , enumerate = const (findSpecialRemotes "borgrepo") + , generate = gen + , configParser = mkRemoteConfigParser + [ optionalStringParser borgrepoField + (FieldDesc "(required) borg repository to use") + ] + , setup = borgSetup + , exportSupported = exportUnsupported + , importSupported = importIsSupported + , thirdPartyPopulated = True + } + +borgrepoField :: RemoteConfigField +borgrepoField = Accepted "borgrepo" + +gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> RemoteStateHandle -> Annex (Maybe Remote) +gen r u rc gc rs = do + c <- parsedRemoteConfig remote rc + cst <- remoteCost gc $ + if borgLocal borgrepo + then nearlyCheapRemoteCost + else expensiveRemoteCost + return $ Just $ Remote + { uuid = u + , cost = cst + , name = Git.repoDescribe r + , storeKey = storeKeyDummy + , retrieveKeyFile = retrieveKeyFileDummy + , retrieveKeyFileCheap = Nothing + -- Borg cryptographically verifies content. + , retrievalSecurityPolicy = RetrievalAllKeysSecure + , removeKey = removeKeyDummy + , lockContent = Nothing + , checkPresent = checkPresentDummy + , checkPresentCheap = borgLocal borgrepo + , exportActions = exportUnsupported + , importActions = importUnsupported + , whereisKey = Nothing + , remoteFsck = Nothing + , repairRepo = Nothing + , config = c + , getRepo = return r + , gitconfig = gc + , localpath = if borgLocal borgrepo && not (null borgrepo) + then Just borgrepo + else Nothing + , remotetype = remote + , availability = if borgLocal borgrepo then LocallyAvailable else GloballyAvailable + , readonly = False + , appendonly = False + , mkUnavailable = return Nothing + , getInfo = return [("repo", borgrepo)] + , claimUrl = Nothing + , checkUrl = Nothing + , remoteStateHandle = rs + } + where + borgrepo = fromMaybe (giveup "missing borgrepo") $ remoteAnnexBorgRepo gc + +borgSetup :: SetupStage -> Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID) +borgSetup _ mu _ c _gc = do + u <- maybe (liftIO genUUID) return mu + + -- verify configuration is sane + let borgrepo = maybe (giveup "Specify borgrepo=") fromProposedAccepted $ + M.lookup borgrepoField c + + -- The borgrepo is stored in git config, as well as this repo's + -- persistant state, so it can vary between hosts. + gitConfigSpecialRemote u c [("borgrepo", borgrepo)] + + -- TODO: untrusted by default, but allow overriding that + + return (c, u) + +borgLocal :: BorgRepo -> Bool +borgLocal = notElem ':' diff --git a/Remote/List.hs b/Remote/List.hs index 7695eec902..8ca9d8f794 100644 --- a/Remote/List.hs +++ b/Remote/List.hs @@ -36,6 +36,7 @@ import qualified Remote.Glacier import qualified Remote.Ddar import qualified Remote.GitLFS import qualified Remote.HttpAlso +import qualified Remote.Borg import qualified Remote.Hook import qualified Remote.External @@ -57,6 +58,7 @@ remoteTypes = map adjustExportImportRemoteType , Remote.Ddar.remote , Remote.GitLFS.remote , Remote.HttpAlso.remote + , Remote.Borg.remote , Remote.Hook.remote , Remote.External.remote ] diff --git a/Types/GitConfig.hs b/Types/GitConfig.hs index cb50c79666..74df213419 100644 --- a/Types/GitConfig.hs +++ b/Types/GitConfig.hs @@ -327,6 +327,7 @@ data RemoteGitConfig = RemoteGitConfig , remoteAnnexGnupgDecryptOptions :: [String] , remoteAnnexRsyncUrl :: Maybe String , remoteAnnexBupRepo :: Maybe String + , remoteAnnexBorgRepo :: Maybe String , remoteAnnexTahoe :: Maybe FilePath , remoteAnnexBupSplitOptions :: [String] , remoteAnnexDirectory :: Maybe FilePath @@ -391,6 +392,7 @@ extractRemoteGitConfig r remotename = do , remoteAnnexGnupgDecryptOptions = getoptions "gnupg-decrypt-options" , remoteAnnexRsyncUrl = notempty $ getmaybe "rsyncurl" , remoteAnnexBupRepo = getmaybe "buprepo" + , remoteAnnexBorgRepo = getmaybe "borgrepo" , remoteAnnexTahoe = getmaybe "tahoe" , remoteAnnexBupSplitOptions = getoptions "bup-split-options" , remoteAnnexDirectory = notempty $ getmaybe "directory" diff --git a/doc/devblog/day_637__thirdparty_of_borg.mdwn b/doc/devblog/day_637__thirdparty_of_borg.mdwn new file mode 100644 index 0000000000..5d4a34c999 --- /dev/null +++ b/doc/devblog/day_637__thirdparty_of_borg.mdwn @@ -0,0 +1,17 @@ +Finally gotten started on the borg special remote idea. A prerequisite of +that is remotes that can be imported from, but not exported to. So I +actually started by allowing setting importtree=yes without +exporttree=yes. A lot of code had assumptions about that not being allowed, +so it took a while to chase down everything. Finished most of that yesterday. + +What I've done today is added a `thirdPartyPopulated` type of remote, +which `git-annex sync` can "pull" from by using the existing import +interface to list files on it, and determine which of them are annex object +files. I have not started on the actual borg remote at all, but this should +be all the groundwork for it done. + +(I also finished up annex.stalldetection earlier this week.) + +--- + +This work was sponsored by Jake Vosloo [on Patreon](https://patreon.com/joeyh). diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 6280bcb864..45306545c7 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -1541,6 +1541,12 @@ Remotes are configured using these settings in `.git/config`. the location of the bup repository to use. Normally this is automatically set up by `git annex initremote`, but you can change it if needed. +* `remote..annex-borgrepo` + + Used by borg special remotes, this configures + the location of the borg repository to use. Normally this is automatically + set up by `git annex initremote`, but you can change it if needed. + * `remote..annex-ddarrepo` Used by ddar special remotes, this configures diff --git a/doc/special_remotes/borg.mdwn b/doc/special_remotes/borg.mdwn index 8869288e88..42f7fefde1 100644 --- a/doc/special_remotes/borg.mdwn +++ b/doc/special_remotes/borg.mdwn @@ -11,7 +11,7 @@ the annexed files that are stored in the borg repository. These parameters can be passed to `git annex initremote` to configure the remote: -* `repository` - The location of a borg repository, eg a path, or +* `borgrepo` - The location of a borg repository, eg a path, or `user@host:path` for ssh access. * `scan` - The path, within the borg repository, to scan for @@ -27,7 +27,7 @@ remote: ## setup example # borg init --encryption=keyfile /path/to/borgrepo - # git annex initremote borg type=borg repository=/path/to/borgrepo scan=`pwd` + # git annex initremote borg type=borg borgrepo=/path/to/borgrepo scan=`pwd` # borg create /path/to/borgrepo `pwd`::{now} # git annex sync borg diff --git a/git-annex.cabal b/git-annex.cabal index 48a1ed5d5c..e035eedd28 100644 --- a/git-annex.cabal +++ b/git-annex.cabal @@ -940,6 +940,7 @@ Executable git-annex Remote Remote.Adb Remote.BitTorrent + Remote.Borg Remote.Bup Remote.Ddar Remote.Directory