From 6f1039900d473c6bdb6d2d378e2ef6b12b44901a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 14 May 2024 13:52:20 -0400 Subject: [PATCH] prevent using git-remote-annex with unsuitable special remote configs I hope to support importtree=yes eventually, but it does not currently work. Added remote..allow-encrypted-gitrepo that needs to be set to allow using it with encrypted git repos. Note that even encryption=pubkey uses a cipher stored in the git repo to encrypt the keys stored in the remote. While it would be possible to not encrypt the GITBUNDLE and GITMANIFEST keys, and then allow using encryption=pubkey, it doesn't currently work, and that would be a complication that I doubt is worth it. --- CmdLine/GitRemoteAnnex.hs | 17 +++++++++++++++- Remote/Helper/Encryptable.hs | 19 ++++++++++------- Types/GitConfig.hs | 6 +++++- doc/git-annex.mdwn | 18 +++++++++++++---- doc/git-remote-annex.mdwn | 5 ----- doc/todo/git-remote-annex.mdwn | 37 +++++++++++++++------------------- 6 files changed, 63 insertions(+), 39 deletions(-) diff --git a/CmdLine/GitRemoteAnnex.hs b/CmdLine/GitRemoteAnnex.hs index e21128e7fd..626b4e4ce3 100644 --- a/CmdLine/GitRemoteAnnex.hs +++ b/CmdLine/GitRemoteAnnex.hs @@ -23,6 +23,7 @@ import qualified Annex.SpecialRemote as SpecialRemote import qualified Annex.Branch import qualified Types.Remote as Remote import qualified Logs.Remote +import Remote.Helper.Encryptable (parseEncryptionMethod) import Annex.Transfer import Backend.GitRemoteAnnex import Config @@ -32,6 +33,7 @@ import Types.ProposedAccepted import Types.Export import Types.GitConfig import Types.Difference +import Types.Crypto import Git.Types import Logs.Difference import Annex.Init @@ -558,8 +560,21 @@ parseManifest b = checkSpecialRemoteProblems :: Remote -> Maybe String checkSpecialRemoteProblems rmt | Remote.thirdPartyPopulated (Remote.remotetype rmt) = - Just "Cannot use this thirdparty-populated special remote as a git remote" + Just $ "Cannot use this thirdparty-populated special" + ++ " remote as a git remote." + | importTree (Remote.config rmt) = + Just $ "Using importtree=yes special remotes as git remotes" + ++ " is not yet supported." + | parseEncryptionMethod (unparsedRemoteConfig (Remote.config rmt)) /= Right NoneEncryption + && not (remoteAnnexAllowEncryptedGitRepo (Remote.gitconfig rmt)) = + Just $ "Using an encrypted special remote as a git" + ++ " remote makes it impossible to clone" + ++ " from it. If you will never need to" + ++ " clone from this remote, set: git config " + ++ decodeBS allowencryptedgitrepo ++ " true" | otherwise = Nothing + where + ConfigKey allowencryptedgitrepo = remoteAnnexConfig rmt "allow-encrypted-gitrepo" -- Downloads the Manifest when present in the remote. When not present, -- returns an empty Manifest. diff --git a/Remote/Helper/Encryptable.hs b/Remote/Helper/Encryptable.hs index 884d53d7bf..9f4bd7fcb1 100644 --- a/Remote/Helper/Encryptable.hs +++ b/Remote/Helper/Encryptable.hs @@ -14,6 +14,7 @@ module Remote.Helper.Encryptable ( encryptionAlreadySetup, encryptionConfigParsers, parseEncryptionConfig, + parseEncryptionMethod, remoteCipher, remoteCipher', embedCreds, @@ -85,7 +86,7 @@ encryptionFieldParser :: RemoteConfigFieldParser encryptionFieldParser = RemoteConfigFieldParser { parserForField = encryptionField , valueParser = \v c -> Just . RemoteConfigValue - <$> parseEncryptionMethod (fmap fromProposedAccepted v) c + <$> parseEncryptionMethod' v c , fieldDesc = FieldDesc "how to encrypt data stored in the special remote" , valueDesc = Just $ ValueDesc $ intercalate " or " (M.keys encryptionMethods) @@ -100,14 +101,18 @@ encryptionMethods = M.fromList , ("sharedpubkey", SharedPubKeyEncryption) ] -parseEncryptionMethod :: Maybe String -> RemoteConfig -> Either String EncryptionMethod -parseEncryptionMethod (Just s) _ = case M.lookup s encryptionMethods of - Just em -> Right em - Nothing -> Left badEncryptionMethod +parseEncryptionMethod :: RemoteConfig -> Either String EncryptionMethod +parseEncryptionMethod c = parseEncryptionMethod' (M.lookup encryptionField c) c + +parseEncryptionMethod' :: Maybe (ProposedAccepted String) -> RemoteConfig -> Either String EncryptionMethod +parseEncryptionMethod' (Just s) _ = + case M.lookup (fromProposedAccepted s) encryptionMethods of + Just em -> Right em + Nothing -> Left badEncryptionMethod -- Hybrid encryption is the default when a keyid is specified without -- an encryption field, or when there's a cipher already but no encryption -- field. -parseEncryptionMethod Nothing c +parseEncryptionMethod' Nothing c | M.member (Accepted "keyid") c || M.member cipherField c = Right HybridEncryption | otherwise = Left badEncryptionMethod @@ -162,7 +167,7 @@ encryptionSetup c gc = do maybe (genCipher pc gpgcmd) (updateCipher pc gpgcmd) (extractCipher pc) where -- The type of encryption - encryption = parseEncryptionMethod (fromProposedAccepted <$> M.lookup encryptionField c) c + encryption = parseEncryptionMethod c -- Generate a new cipher, depending on the chosen encryption scheme genCipher pc gpgcmd = case encryption of Right NoneEncryption -> return (c, NoEncryption) diff --git a/Types/GitConfig.hs b/Types/GitConfig.hs index 26540b8484..49fde98fc3 100644 --- a/Types/GitConfig.hs +++ b/Types/GitConfig.hs @@ -362,6 +362,7 @@ data RemoteGitConfig = RemoteGitConfig , remoteAnnexStopCommand :: Maybe String , remoteAnnexSpeculatePresent :: Bool , remoteAnnexBare :: Maybe Bool + , remoteAnnexAllowEncryptedGitRepo :: Bool , remoteAnnexRetry :: Maybe Integer , remoteAnnexForwardRetry :: Maybe Integer , remoteAnnexRetryDelay :: Maybe Seconds @@ -430,8 +431,11 @@ extractRemoteGitConfig r remotename = do , remoteAnnexTrustLevel = notempty $ getmaybe "trustlevel" , remoteAnnexStartCommand = notempty $ getmaybe "start-command" , remoteAnnexStopCommand = notempty $ getmaybe "stop-command" - , remoteAnnexSpeculatePresent = getbool "speculate-present" False + , remoteAnnexSpeculatePresent = + getbool "speculate-present" False , remoteAnnexBare = getmaybebool "bare" + , remoteAnnexAllowEncryptedGitRepo = + getbool "allow-encrypted-gitrepo" False , remoteAnnexRetry = getmayberead "retry" , remoteAnnexForwardRetry = getmayberead "forward-retry" , remoteAnnexRetryDelay = Seconds diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index c8581ed713..561c8c1836 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -1634,10 +1634,6 @@ Remotes are configured using these settings in `.git/config`. configured by the trust and untrust commands. The value can be any of "trusted", "semitrusted" or "untrusted". -* `remote..annex-availability` - - This configuration setting is no longer used. - * `remote..annex-speculate-present` Set to "true" to make git-annex speculate that this remote may contain the @@ -1663,11 +1659,25 @@ Remotes are configured using these settings in `.git/config`. while preventing a new clone needing to download too many objects. Set to 0 to disable re-pushing. +* `remote..allow-encrypted-gitrepo` + + Setting this to true allows using [[git-remote-annex]] to push the git + repository to an encrypted special remote. + + That is not allowed by default, because it is impossible to git clone + from an encrypted special remote, since it needs encryption keys stored + in the remote. So take care that, if you set this, you don't rely + on the encrypted special remote being the only copy of your git repository. + * `remote..annex-bare` Can be used to tell git-annex if a remote is a bare repository or not. Normally, git-annex determines this automatically. +* `remote..annex-availability` + + This configuration setting is no longer used. + * `remote..annex-ssh-options` Options to use when using ssh to talk to this remote. diff --git a/doc/git-remote-annex.mdwn b/doc/git-remote-annex.mdwn index 4fefb1bd36..3da0961c5b 100644 --- a/doc/git-remote-annex.mdwn +++ b/doc/git-remote-annex.mdwn @@ -12,11 +12,6 @@ This is a git remote helper program that allows git to clone, pull and push from a git repository that is stored in a git-annex special remote. -It can be used with any special remote except those that use -encryption=shared or encryption=hybrid. (Since those types of encryption -rely on a cipher that is checked into the git repository, cloning from -such a special remote would present a chicken and egg problem.) - The format of the remote URL is "annex::" followed by the UUID of the special remote, and then followed by all of the configuration parameters of the special remote. diff --git a/doc/todo/git-remote-annex.mdwn b/doc/todo/git-remote-annex.mdwn index 7d115739cc..6863f56655 100644 --- a/doc/todo/git-remote-annex.mdwn +++ b/doc/todo/git-remote-annex.mdwn @@ -1,5 +1,5 @@ -git-remote-annex will be a program that allows push/pull of a git -repository to any git-annex special remote. +git-remote-annex will be a program that allows push/pull/clone of a git +repository to many types of git-annex special remote. This is a redesign and reimplementation of git-remote-datalad-annex. It will be a safer implementation, will support incremental pushes, and @@ -23,14 +23,20 @@ This is implememented and working. Remaining todo list for it: * Need to mention git-remote-annex in special remotes page, and perhaps write a tip for it. Also link to it from git-annex man page. -* initremote could optionally configure the url to a special remote - to an annex:: url. This would make it easier to use git-remote-annex, - since the user would not need to set up the url themselves. - (Also it would then avoid setting `skipFetchAll = true`) +* It would be nice if git-annex could generate an annex:: url + for a special remote and show it to the user, eg when + they have set the shorthand "annex::" url, so they know the full url. + `git-annex info $remote` could also display it. + Currently, the user has to remember how the special remote was + configured and replicate it all in the url. -* Prevent using with remotes that are encrypted using a cipher - stored in the repo. Chicken and egg problem cloning from - such a remote. Maybe allow advanced users to force it? + There are some difficulties to doing this, including that + RemoteConfig can have hidden fields that should be omitted. + +* initremote/enableremote could have an option that configures the url to a + special remote to a annex:: url. This would make it easier to use + git-remote-annex, since the user would not need to set up the url + themselves. (Also it would then avoid setting `skipFetchAll = true`) * Improve recovery from interrupted push by using outManifest to clean up after it. (Requires populating outManifest.) @@ -47,18 +53,6 @@ This is implememented and working. Remaining todo list for it: `datalad-annex::https://example.com?type=web&url={noquery}` Supporting something like this would be good. -* It would be nice if git-annex could generate an annex:: url - for a special remote and show it to the user, eg when - they have set the shorthand "annex::" url, so they know the full url. - `git-annex info $remote` could also display it. - Currently, the user has to remember how the special remote was - configured and replicate it all in the url. - - There are some difficulties to doing this, including that - RemoteConfig can have hidden fields that should be omitted, - and that some, like type=directory, remove some configs - (eg directory=) in their setup action. - * Improve behavior in push races. A race can overwrite a change to the MANIFEST and lose work that was pushed from the other repo. From the user's perspective, that situation is the same as if one repo @@ -101,3 +95,4 @@ This is implememented and working. Remaining todo list for it: Also, when the remote uses importree=yes, pushing to it updates content identifiers, which currently get recorded in the git-annex branch. It would be good to avoid that being written as well. +