2013-01-01 17:52:47 +00:00
|
|
|
{- git-annex configuration
|
|
|
|
|
-
|
2021-04-05 19:21:20 +00:00
|
|
|
- Copyright 2012-2021 Joey Hess <id@joeyh.name>
|
2013-01-01 17:52:47 +00:00
|
|
|
-
|
2019-03-13 19:48:14 +00:00
|
|
|
- Licensed under the GNU AGPL version 3 or higher.
|
2013-01-01 17:52:47 +00:00
|
|
|
-}
|
|
|
|
|
|
2020-03-02 19:50:40 +00:00
|
|
|
{-# LANGUAGE TypeSynonymInstances, FlexibleInstances #-}
|
2019-11-27 20:54:11 +00:00
|
|
|
{-# LANGUAGE OverloadedStrings #-}
|
|
|
|
|
|
2013-01-01 17:52:47 +00:00
|
|
|
module Types.GitConfig (
|
2020-09-16 15:16:48 +00:00
|
|
|
GlobalConfigurable(..),
|
git-annex config annex.largefiles
annex.largefiles can be configured by git-annex config, to more easily set
a default that will also be used by clones, without needing to shoehorn the
expression into the gitattributes file. The git config and gitattributes
override that.
Whenever something is added to git-annex config, we have to consider what
happens if a user puts a purposfully bad value in there. Or, if a new
git-annex adds some new value that an old git-annex can't parse.
In this case, a global annex.largefiles that can't be parsed currently
makes an error be thrown. That might not be ideal, but the gitattribute
behaves the same, and is almost equally repo-global.
Performance notes:
git-annex add and addurl construct a matcher once
and uses it for every file, so the added time penalty for reading the global
config log is minor. If the gitattributes annex.largefiles were deprecated,
git-annex add would get around 2% faster (excluding hashing), because
looking that up for each file is not fast. So this new way of setting
it is progress toward speeding up add.
git-annex smudge does need to load the log every time. As well as checking
the git attribute. Not ideal. Setting annex.gitaddtoannex=false avoids
both overheads.
2019-12-20 16:12:31 +00:00
|
|
|
ConfigSource(..),
|
2013-01-01 17:52:47 +00:00
|
|
|
GitConfig(..),
|
|
|
|
|
extractGitConfig,
|
2017-02-03 17:40:14 +00:00
|
|
|
mergeGitConfig,
|
2020-03-02 19:50:40 +00:00
|
|
|
globalConfigs,
|
2013-01-01 17:52:47 +00:00
|
|
|
RemoteGitConfig(..),
|
|
|
|
|
extractRemoteGitConfig,
|
2017-08-17 16:26:14 +00:00
|
|
|
dummyRemoteGitConfig,
|
2020-03-02 19:50:40 +00:00
|
|
|
annexConfig,
|
|
|
|
|
RemoteNameable(..),
|
|
|
|
|
remoteAnnexConfig,
|
|
|
|
|
remoteConfig,
|
2013-01-01 17:52:47 +00:00
|
|
|
) where
|
|
|
|
|
|
|
|
|
|
import Common
|
|
|
|
|
import qualified Git
|
|
|
|
|
import qualified Git.Config
|
2015-02-12 19:44:10 +00:00
|
|
|
import qualified Git.Construct
|
removal of the rest of remoteGitConfig
In keyUrls, the GitConfig is used only by annexLocations
to support configured Differences. Since such configurations affect all
clones of a repository, the local repo's GitConfig must have the same
information as the remote's GitConfig would have. So, used getGitConfig
to get the local GitConfig, which is cached and so available cheaply.
That actually fixed a bug noone had ever noticed: keyUrls is
used for remotes accessed over http. The full git config of such a
remote is normally not available, so the remoteGitConfig that keyUrls
used would not have the necessary information in it.
In copyFromRemoteCheap', it uses gitAnnexLocation,
which does need the GitConfig of the remote repo itself in order to
check if it's crippled, supports symlinks, etc. So, made the
State include that GitConfig, cached. The use of gitAnnexLocation is
within a (not $ Git.repoIsUrl repo) guard, so it's local, and so
its git config will always be read and available.
(Note that gitAnnexLocation in turn calls annexLocations, so the
Differences config it uses in this case comes from the remote repo's
GitConfig and not from the local repo's GitConfig. As explained above
this is ok since they must have the same value.)
Not very happy with this mess of different GitConfigs not type-safe and
some read only sometimes etc. Very hairy. Think I got it this change
right. Test suite passes..
This commit was sponsored by Ethan Aubin.
2018-06-05 18:23:34 +00:00
|
|
|
import Git.Types
|
2017-02-17 18:04:43 +00:00
|
|
|
import Git.ConfigTypes
|
2021-04-23 18:21:57 +00:00
|
|
|
import Git.Remote (isRemoteKey, remoteKeyToRemoteName)
|
2019-11-11 22:20:35 +00:00
|
|
|
import Git.Branch (CommitMode(..))
|
2023-04-12 21:18:29 +00:00
|
|
|
import Git.Quote (QuotePath(..))
|
2013-01-01 17:52:47 +00:00
|
|
|
import Utility.DataUnits
|
2013-03-13 20:16:01 +00:00
|
|
|
import Config.Cost
|
2016-01-20 20:55:06 +00:00
|
|
|
import Types.UUID
|
2013-11-22 20:04:20 +00:00
|
|
|
import Types.Distribution
|
2018-10-04 16:47:27 +00:00
|
|
|
import Types.Concurrency
|
2014-01-21 21:08:49 +00:00
|
|
|
import Types.NumCopies
|
2015-01-27 21:38:06 +00:00
|
|
|
import Types.Difference
|
2015-05-14 19:44:08 +00:00
|
|
|
import Types.RefSpec
|
2018-10-25 21:23:53 +00:00
|
|
|
import Types.RepoVersion
|
2020-12-08 19:22:18 +00:00
|
|
|
import Types.StallDetection
|
2023-02-07 20:28:46 +00:00
|
|
|
import Types.View
|
2017-08-17 16:26:14 +00:00
|
|
|
import Config.DynamicConfig
|
assistant unused file handling
Make sanity checker run git annex unused daily, and queue up transfers
of unused files to any remotes that will have them. The transfer retrying
code works for us here, so eg when a backup disk remote is plugged in,
any transfers to it are done. Once the unused files reach a remote,
they'll be removed locally as unwanted.
If the setup does not cause unused files to go to a remote, they'll pile
up, and the sanity checker detects this using some heuristics that are
pretty good -- 1000 unused files, or 10% of disk used by unused files,
or more disk wasted by unused files than is left free. Once it detects
this, it pops up an alert in the webapp, with a button to take action.
TODO: Webapp UI to configure this, and also the ability to launch an
immediate cleanup of all unused files.
This commit was sponsored by Simon Michael.
2014-01-23 02:48:56 +00:00
|
|
|
import Utility.HumanTime
|
2015-09-09 22:06:49 +00:00
|
|
|
import Utility.Gpg (GpgCmd, mkGpgCmd)
|
2015-11-12 21:47:31 +00:00
|
|
|
import Utility.ThreadScheduler (Seconds(..))
|
limit url downloads to whitelisted schemes
Security fix! Allowing any schemes, particularly file: and
possibly others like scp: allowed file exfiltration by anyone who had
write access to the git repository, since they could add an annexed file
using such an url, or using an url that redirected to such an url,
and wait for the victim to get it into their repository and send them a copy.
* Added annex.security.allowed-url-schemes setting, which defaults
to only allowing http and https URLs. Note especially that file:/
is no longer enabled by default.
* Removed annex.web-download-command, since its interface does not allow
supporting annex.security.allowed-url-schemes across redirects.
If you used this setting, you may want to instead use annex.web-options
to pass options to curl.
With annex.web-download-command removed, nearly all url accesses in
git-annex are made via Utility.Url via http-client or curl. http-client
only supports http and https, so no problem there.
(Disabling one and not the other is not implemented.)
Used curl --proto to limit the allowed url schemes.
Note that this will cause git annex fsck --from web to mark files using
a disallowed url scheme as not being present in the web. That seems
acceptable; fsck --from web also does that when a web server is not available.
youtube-dl already disabled file: itself (probably for similar
reasons). The scheme check was also added to youtube-dl urls for
completeness, although that check won't catch any redirects it might
follow. But youtube-dl goes off and does its own thing with other
protocols anyway, so that's fine.
Special remotes that support other domain-specific url schemes are not
affected by this change. In the bittorrent remote, aria2c can still
download magnet: links. The download of the .torrent file is
otherwise now limited by annex.security.allowed-url-schemes.
This does not address any external special remotes that might download
an url themselves. Current thinking is all external special remotes will
need to be audited for this problem, although many of them will use
http libraries that only support http and not curl's menagarie.
The related problem of accessing private localhost and LAN urls is not
addressed by this commit.
This commit was sponsored by Brett Eisenberg on Patreon.
2018-06-15 20:52:24 +00:00
|
|
|
import Utility.Url (Scheme, mkScheme)
|
2013-01-01 17:52:47 +00:00
|
|
|
|
2017-08-17 16:26:14 +00:00
|
|
|
import Control.Concurrent.STM
|
limit url downloads to whitelisted schemes
Security fix! Allowing any schemes, particularly file: and
possibly others like scp: allowed file exfiltration by anyone who had
write access to the git repository, since they could add an annexed file
using such an url, or using an url that redirected to such an url,
and wait for the victim to get it into their repository and send them a copy.
* Added annex.security.allowed-url-schemes setting, which defaults
to only allowing http and https URLs. Note especially that file:/
is no longer enabled by default.
* Removed annex.web-download-command, since its interface does not allow
supporting annex.security.allowed-url-schemes across redirects.
If you used this setting, you may want to instead use annex.web-options
to pass options to curl.
With annex.web-download-command removed, nearly all url accesses in
git-annex are made via Utility.Url via http-client or curl. http-client
only supports http and https, so no problem there.
(Disabling one and not the other is not implemented.)
Used curl --proto to limit the allowed url schemes.
Note that this will cause git annex fsck --from web to mark files using
a disallowed url scheme as not being present in the web. That seems
acceptable; fsck --from web also does that when a web server is not available.
youtube-dl already disabled file: itself (probably for similar
reasons). The scheme check was also added to youtube-dl urls for
completeness, although that check won't catch any redirects it might
follow. But youtube-dl goes off and does its own thing with other
protocols anyway, so that's fine.
Special remotes that support other domain-specific url schemes are not
affected by this change. In the bittorrent remote, aria2c can still
download magnet: links. The download of the .torrent file is
otherwise now limited by annex.security.allowed-url-schemes.
This does not address any external special remotes that might download
an url themselves. Current thinking is all external special remotes will
need to be audited for this problem, although many of them will use
http libraries that only support http and not curl's menagarie.
The related problem of accessing private localhost and LAN urls is not
addressed by this commit.
This commit was sponsored by Brett Eisenberg on Patreon.
2018-06-15 20:52:24 +00:00
|
|
|
import qualified Data.Set as S
|
2021-04-23 18:21:57 +00:00
|
|
|
import qualified Data.Map as M
|
2020-03-02 19:50:40 +00:00
|
|
|
import qualified Data.ByteString as B
|
2022-08-12 17:18:15 +00:00
|
|
|
import qualified System.FilePath.ByteString as P
|
2017-08-17 16:26:14 +00:00
|
|
|
|
|
|
|
|
-- | A configurable value, that may not be fully determined yet because
|
|
|
|
|
-- the global git config has not yet been loaded.
|
2020-09-16 15:16:48 +00:00
|
|
|
data GlobalConfigurable a
|
git-annex config annex.largefiles
annex.largefiles can be configured by git-annex config, to more easily set
a default that will also be used by clones, without needing to shoehorn the
expression into the gitattributes file. The git config and gitattributes
override that.
Whenever something is added to git-annex config, we have to consider what
happens if a user puts a purposfully bad value in there. Or, if a new
git-annex adds some new value that an old git-annex can't parse.
In this case, a global annex.largefiles that can't be parsed currently
makes an error be thrown. That might not be ideal, but the gitattribute
behaves the same, and is almost equally repo-global.
Performance notes:
git-annex add and addurl construct a matcher once
and uses it for every file, so the added time penalty for reading the global
config log is minor. If the gitattributes annex.largefiles were deprecated,
git-annex add would get around 2% faster (excluding hashing), because
looking that up for each file is not fast. So this new way of setting
it is progress toward speeding up add.
git-annex smudge does need to load the log every time. As well as checking
the git attribute. Not ideal. Setting annex.gitaddtoannex=false avoids
both overheads.
2019-12-20 16:12:31 +00:00
|
|
|
= HasGitConfig a
|
|
|
|
|
-- ^ The git config has a value.
|
|
|
|
|
| HasGlobalConfig a
|
|
|
|
|
-- ^ The global config has a value (and the git config does not).
|
2017-02-03 17:40:14 +00:00
|
|
|
| DefaultConfig a
|
|
|
|
|
-- ^ A default value is known, but not all config sources
|
|
|
|
|
-- have been read yet.
|
2017-02-03 18:31:17 +00:00
|
|
|
deriving (Show)
|
2017-02-03 17:40:14 +00:00
|
|
|
|
git-annex config annex.largefiles
annex.largefiles can be configured by git-annex config, to more easily set
a default that will also be used by clones, without needing to shoehorn the
expression into the gitattributes file. The git config and gitattributes
override that.
Whenever something is added to git-annex config, we have to consider what
happens if a user puts a purposfully bad value in there. Or, if a new
git-annex adds some new value that an old git-annex can't parse.
In this case, a global annex.largefiles that can't be parsed currently
makes an error be thrown. That might not be ideal, but the gitattribute
behaves the same, and is almost equally repo-global.
Performance notes:
git-annex add and addurl construct a matcher once
and uses it for every file, so the added time penalty for reading the global
config log is minor. If the gitattributes annex.largefiles were deprecated,
git-annex add would get around 2% faster (excluding hashing), because
looking that up for each file is not fast. So this new way of setting
it is progress toward speeding up add.
git-annex smudge does need to load the log every time. As well as checking
the git attribute. Not ideal. Setting annex.gitaddtoannex=false avoids
both overheads.
2019-12-20 16:12:31 +00:00
|
|
|
data ConfigSource = FromGitConfig | FromGlobalConfig
|
|
|
|
|
|
2013-01-01 17:52:47 +00:00
|
|
|
{- Main git-annex settings. Each setting corresponds to a git-config key
|
|
|
|
|
- such as annex.foo -}
|
|
|
|
|
data GitConfig = GitConfig
|
2018-10-25 21:23:53 +00:00
|
|
|
{ annexVersion :: Maybe RepoVersion
|
2016-01-20 20:55:06 +00:00
|
|
|
, annexUUID :: UUID
|
2014-01-21 21:08:49 +00:00
|
|
|
, annexNumCopies :: Maybe NumCopies
|
2013-01-01 17:52:47 +00:00
|
|
|
, annexDiskReserve :: Integer
|
|
|
|
|
, annexDirect :: Bool
|
2017-05-09 19:04:07 +00:00
|
|
|
, annexBackend :: Maybe String
|
2013-01-01 17:52:47 +00:00
|
|
|
, annexQueueSize :: Maybe Int
|
|
|
|
|
, annexBloomCapacity :: Maybe Int
|
|
|
|
|
, annexBloomAccuracy :: Maybe Int
|
|
|
|
|
, annexSshCaching :: Maybe Bool
|
|
|
|
|
, annexAlwaysCommit :: Bool
|
2022-07-18 19:50:36 +00:00
|
|
|
, annexAlwaysCompact :: Bool
|
2018-08-02 18:06:06 +00:00
|
|
|
, annexCommitMessage :: Maybe String
|
2018-02-22 18:25:32 +00:00
|
|
|
, annexMergeAnnexBranches :: Bool
|
2013-01-01 17:52:47 +00:00
|
|
|
, annexDelayAdd :: Maybe Int
|
|
|
|
|
, annexHttpHeaders :: [String]
|
|
|
|
|
, annexHttpHeadersCommand :: Maybe String
|
2020-09-16 15:16:48 +00:00
|
|
|
, annexAutoCommit :: GlobalConfigurable Bool
|
|
|
|
|
, annexResolveMerge :: GlobalConfigurable Bool
|
git-annex pull and push
Split out two new commands, git-annex pull and git-annex push. Those plus a
git commit are equivilant to git-annex sync.
In a sense, git-annex sync conflates 3 things, and it would have been
better to have push and pull from the beginning and not sync. Although
note that git-annex sync --content is faster than a pull followed by a
push, because it only has to walk the tree once, look at preferred
content once, etc. So there is some value in git-annex sync in speed, as
well as user convenience.
And it would be hard to split out pull and push from sync, as far as the
implementaton goes. The implementation inside sync was easy, just adjust
SyncOptions so it does the right thing.
Note that the new commands default to syncing content, unless
annex.synccontent is explicitly set to false. I'd like sync to also do
that, but that's a hard transition to make. As a start to that
transition, I added a note to git-annex-sync.mdwn that it may start to
do so in a future version of git-annex. But a real transition would
necessarily involve displaying warnings when sync is used without
--content, and time.
Sponsored-by: Kevin Mueller on Patreon
2023-05-16 20:37:30 +00:00
|
|
|
, annexSyncContent :: GlobalConfigurable (Maybe Bool)
|
2020-09-16 15:16:48 +00:00
|
|
|
, annexSyncOnlyAnnex :: GlobalConfigurable Bool
|
2013-06-18 00:41:17 +00:00
|
|
|
, annexDebug :: Bool
|
2021-04-06 19:14:00 +00:00
|
|
|
, annexDebugFilter :: Maybe String
|
2013-01-27 13:33:19 +00:00
|
|
|
, annexWebOptions :: [String]
|
2017-11-29 19:49:05 +00:00
|
|
|
, annexYoutubeDlOptions :: [String]
|
2021-08-27 13:44:23 +00:00
|
|
|
, annexYoutubeDlCommand :: Maybe String
|
2014-12-17 03:22:46 +00:00
|
|
|
, annexAriaTorrentOptions :: [String]
|
2013-02-14 18:10:36 +00:00
|
|
|
, annexCrippledFileSystem :: Bool
|
2020-09-16 15:16:48 +00:00
|
|
|
, annexLargeFiles :: GlobalConfigurable (Maybe String)
|
|
|
|
|
, annexDotFiles :: GlobalConfigurable Bool
|
2019-10-23 19:20:00 +00:00
|
|
|
, annexGitAddToAnnex :: Bool
|
2016-01-28 18:04:32 +00:00
|
|
|
, annexAddSmallFiles :: Bool
|
2013-10-29 20:48:06 +00:00
|
|
|
, annexFsckNudge :: Bool
|
2013-11-22 20:04:20 +00:00
|
|
|
, annexAutoUpgrade :: AutoUpgrade
|
2014-01-23 03:10:51 +00:00
|
|
|
, annexExpireUnused :: Maybe (Maybe Duration)
|
Added annex.freezecontent-command and annex.thawcontent-command configs
Freeze first sets the file perms, and then runs
freezecontent-command. Thaw runs thawcontent-command before
restoring file permissions. This is in case the freeze command
prevents changing file perms, as eg setting a file immutable does.
Also, changing file perms tends to mess up previously set ACLs.
git-annex init's probe for crippled filesystem uses them, so if file perms
don't work, but freezecontent-command manages to prevent write to a file,
it won't treat the filesystem as crippled.
When the the filesystem has been probed as crippled, the hooks are not
used, because there seems to be no point then; git-annex won't be relying
on locking annex objects down. Also, this avoids them being run when the
file perms have not been changed, in case they somehow rely on
git-annex's setting of the file perms in order to work.
Sponsored-by: Dartmouth College's Datalad project
2021-06-21 18:40:20 +00:00
|
|
|
, annexFreezeContentCommand :: Maybe String
|
|
|
|
|
, annexThawContentCommand :: Maybe String
|
2014-01-24 16:58:52 +00:00
|
|
|
, annexSecureEraseCommand :: Maybe String
|
2014-02-23 04:08:29 +00:00
|
|
|
, annexGenMetaData :: Bool
|
2014-03-01 04:31:17 +00:00
|
|
|
, annexListen :: Maybe String
|
2014-03-05 21:44:14 +00:00
|
|
|
, annexStartupScan :: Bool
|
2014-09-05 17:44:09 +00:00
|
|
|
, annexHardLink :: Bool
|
2015-12-27 19:59:59 +00:00
|
|
|
, annexThin :: Bool
|
2015-09-09 22:06:49 +00:00
|
|
|
, annexDifferences :: Differences
|
|
|
|
|
, annexUsedRefSpec :: Maybe RefSpec
|
Do verification of checksums of annex objects downloaded from remotes.
* When annex objects are received into git repositories, their checksums are
verified then too.
* To get the old, faster, behavior of not verifying checksums, set
annex.verify=false, or remote.<name>.annex-verify=false.
* setkey, rekey: These commands also now verify that the provided file
matches the key, unless annex.verify=false.
* reinject: Already verified content; this can now be disabled by
setting annex.verify=false.
recvkey and reinject already did verification, so removed now duplicate
code from them. fsck still does its own verification, which is ok since it
does not use getViaTmp, so verification doesn't happen twice when using fsck
--from.
2015-10-01 19:54:37 +00:00
|
|
|
, annexVerify :: Bool
|
2015-11-12 21:47:31 +00:00
|
|
|
, annexPidLock :: Bool
|
|
|
|
|
, annexPidLockTimeout :: Seconds
|
2022-08-11 20:57:44 +00:00
|
|
|
, annexDbDir :: Maybe RawFilePath
|
2020-09-16 15:16:48 +00:00
|
|
|
, annexAddUnlocked :: GlobalConfigurable (Maybe String)
|
annex.securehashesonly
Cryptographically secure hashes can be forced to be used in a repository,
by setting annex.securehashesonly. This does not prevent the git repository
from containing files with insecure hashes, but it does prevent the content
of such files from being pulled into .git/annex/objects from another
repository.
We want to make sure that at no point does git-annex accept content into
.git/annex/objects that is hashed with an insecure key. Here's how it
was done:
* .git/annex/objects/xx/yy/KEY/ is kept frozen, so nothing can be
written to it normally
* So every place that writes content must call, thawContent or modifyContent.
We can audit for these, and be sure we've considered all cases.
* The main functions are moveAnnex, and linkToAnnex; these were made to
check annex.securehashesonly, and are the main security boundary
for annex.securehashesonly.
* Most other calls to modifyContent deal with other files in the KEY
directory (inode cache etc). The other ones that mess with the content
are:
- Annex.Direct.toDirectGen, in which content already in the
annex directory is moved to the direct mode file, so not relevant.
- fix and lock, which don't add new content
- Command.ReKey.linkKey, which manually unlocks it to make a
copy.
* All other calls to thawContent appear safe.
Made moveAnnex return a Bool, so checked all callsites and made them
deal with a failure in appropriate ways.
linkToAnnex simply returns LinkAnnexFailed; all callsites already deal
with it failing in appropriate ways.
This commit was sponsored by Riku Voipio.
2017-02-27 17:01:32 +00:00
|
|
|
, annexSecureHashesOnly :: Bool
|
2018-03-24 14:37:25 +00:00
|
|
|
, annexRetry :: Maybe Integer
|
2020-09-04 19:16:40 +00:00
|
|
|
, annexForwardRetry :: Maybe Integer
|
2018-03-24 14:37:25 +00:00
|
|
|
, annexRetryDelay :: Maybe Seconds
|
limit url downloads to whitelisted schemes
Security fix! Allowing any schemes, particularly file: and
possibly others like scp: allowed file exfiltration by anyone who had
write access to the git repository, since they could add an annexed file
using such an url, or using an url that redirected to such an url,
and wait for the victim to get it into their repository and send them a copy.
* Added annex.security.allowed-url-schemes setting, which defaults
to only allowing http and https URLs. Note especially that file:/
is no longer enabled by default.
* Removed annex.web-download-command, since its interface does not allow
supporting annex.security.allowed-url-schemes across redirects.
If you used this setting, you may want to instead use annex.web-options
to pass options to curl.
With annex.web-download-command removed, nearly all url accesses in
git-annex are made via Utility.Url via http-client or curl. http-client
only supports http and https, so no problem there.
(Disabling one and not the other is not implemented.)
Used curl --proto to limit the allowed url schemes.
Note that this will cause git annex fsck --from web to mark files using
a disallowed url scheme as not being present in the web. That seems
acceptable; fsck --from web also does that when a web server is not available.
youtube-dl already disabled file: itself (probably for similar
reasons). The scheme check was also added to youtube-dl urls for
completeness, although that check won't catch any redirects it might
follow. But youtube-dl goes off and does its own thing with other
protocols anyway, so that's fine.
Special remotes that support other domain-specific url schemes are not
affected by this change. In the bittorrent remote, aria2c can still
download magnet: links. The download of the .torrent file is
otherwise now limited by annex.security.allowed-url-schemes.
This does not address any external special remotes that might download
an url themselves. Current thinking is all external special remotes will
need to be audited for this problem, although many of them will use
http libraries that only support http and not curl's menagarie.
The related problem of accessing private localhost and LAN urls is not
addressed by this commit.
This commit was sponsored by Brett Eisenberg on Patreon.
2018-06-15 20:52:24 +00:00
|
|
|
, annexAllowedUrlSchemes :: S.Set Scheme
|
2019-05-30 16:43:40 +00:00
|
|
|
, annexAllowedIPAddresses :: String
|
2018-06-21 17:34:11 +00:00
|
|
|
, annexAllowUnverifiedDownloads :: Bool
|
2018-09-24 16:07:46 +00:00
|
|
|
, annexMaxExtensionLength :: Maybe Int
|
2018-10-04 16:47:27 +00:00
|
|
|
, annexJobs :: Concurrency
|
2018-12-04 18:16:56 +00:00
|
|
|
, annexCacheCreds :: Bool
|
2019-09-01 17:29:55 +00:00
|
|
|
, annexAutoUpgradeRepository :: Bool
|
2019-11-11 22:20:35 +00:00
|
|
|
, annexCommitMode :: CommitMode
|
2020-05-28 19:55:17 +00:00
|
|
|
, annexSkipUnknown :: Bool
|
2020-11-16 18:09:55 +00:00
|
|
|
, annexAdjustedBranchRefresh :: Integer
|
2021-03-23 18:04:34 +00:00
|
|
|
, annexSupportUnlocked :: Bool
|
2013-02-15 20:02:35 +00:00
|
|
|
, coreSymlinks :: Bool
|
2015-05-19 19:04:24 +00:00
|
|
|
, coreSharedRepository :: SharedRepository
|
git style filename quoting controlled by core.quotePath
This is by no means complete, but escaping filenames in actionItemDesc does
cover most commands.
Note that for ActionItemBranchFilePath, the value is branch:file, and I
choose to only quote the file part (if necessary). I considered quoting the
whole thing. But, branch names cannot contain control characters, and while
they can contain unicode, git coes not quote unicode when displaying branch
names. So, it would be surprising for git-annex to quote unicode in a
branch name.
The find command is the most obvious command that still needs to be
dealt with. There are probably other places that filenames also get
displayed, eg embedded in error messages.
Some other commands use ActionItemOther with a filename, I think that
ActionItemOther should either be pre-sanitized, or should explicitly not
be used for filenames, so that needs more work.
When --json is used, unicode does not get escaped, but control
characters were already escaped in json.
(Key escaping may turn out to be needed, but I'm ignoring that for now.)
Sponsored-by: unqueued on Patreon
2023-04-08 18:20:02 +00:00
|
|
|
, coreQuotePath :: QuotePath
|
2017-02-17 18:04:43 +00:00
|
|
|
, receiveDenyCurrentBranch :: DenyCurrentBranch
|
2013-09-24 21:25:47 +00:00
|
|
|
, gcryptId :: Maybe String
|
2015-09-09 22:06:49 +00:00
|
|
|
, gpgCmd :: GpgCmd
|
2020-09-07 17:50:58 +00:00
|
|
|
, mergeDirectoryRenames :: Maybe String
|
2021-04-23 18:21:57 +00:00
|
|
|
, annexPrivateRepos :: S.Set UUID
|
2021-05-27 16:37:39 +00:00
|
|
|
, annexAdviceNoSshCaching :: Bool
|
2023-02-07 20:28:46 +00:00
|
|
|
, annexViewUnsetDirectory :: ViewUnset
|
2013-01-01 17:52:47 +00:00
|
|
|
}
|
|
|
|
|
|
git-annex config annex.largefiles
annex.largefiles can be configured by git-annex config, to more easily set
a default that will also be used by clones, without needing to shoehorn the
expression into the gitattributes file. The git config and gitattributes
override that.
Whenever something is added to git-annex config, we have to consider what
happens if a user puts a purposfully bad value in there. Or, if a new
git-annex adds some new value that an old git-annex can't parse.
In this case, a global annex.largefiles that can't be parsed currently
makes an error be thrown. That might not be ideal, but the gitattribute
behaves the same, and is almost equally repo-global.
Performance notes:
git-annex add and addurl construct a matcher once
and uses it for every file, so the added time penalty for reading the global
config log is minor. If the gitattributes annex.largefiles were deprecated,
git-annex add would get around 2% faster (excluding hashing), because
looking that up for each file is not fast. So this new way of setting
it is progress toward speeding up add.
git-annex smudge does need to load the log every time. As well as checking
the git attribute. Not ideal. Setting annex.gitaddtoannex=false avoids
both overheads.
2019-12-20 16:12:31 +00:00
|
|
|
extractGitConfig :: ConfigSource -> Git.Repo -> GitConfig
|
|
|
|
|
extractGitConfig configsource r = GitConfig
|
2020-03-02 19:50:40 +00:00
|
|
|
{ annexVersion = RepoVersion <$> getmayberead (annexConfig "version")
|
2021-04-23 18:21:57 +00:00
|
|
|
, annexUUID = hereuuid
|
2022-03-28 19:19:52 +00:00
|
|
|
, annexNumCopies = configuredNumCopies
|
|
|
|
|
<$> getmayberead (annexConfig "numcopies")
|
2022-09-21 19:00:13 +00:00
|
|
|
, annexDiskReserve = fromMaybe (onemegabyte * 100) $
|
2020-03-02 19:50:40 +00:00
|
|
|
readSize dataUnits =<< getmaybe (annexConfig "diskreserve")
|
|
|
|
|
, annexDirect = getbool (annexConfig "direct") False
|
2017-05-09 19:04:07 +00:00
|
|
|
, annexBackend = maybe
|
|
|
|
|
-- annex.backends is the old name of the option, still used
|
|
|
|
|
-- when annex.backend is not set.
|
2020-03-02 19:50:40 +00:00
|
|
|
(headMaybe $ getwords (annexConfig "backends"))
|
2017-05-09 19:04:07 +00:00
|
|
|
Just
|
2020-03-02 19:50:40 +00:00
|
|
|
(getmaybe (annexConfig "backend"))
|
|
|
|
|
, annexQueueSize = getmayberead (annexConfig "queuesize")
|
|
|
|
|
, annexBloomCapacity = getmayberead (annexConfig "bloomcapacity")
|
|
|
|
|
, annexBloomAccuracy = getmayberead (annexConfig "bloomaccuracy")
|
|
|
|
|
, annexSshCaching = getmaybebool (annexConfig "sshcaching")
|
|
|
|
|
, annexAlwaysCommit = getbool (annexConfig "alwayscommit") True
|
2022-07-18 19:50:36 +00:00
|
|
|
, annexAlwaysCompact = getbool (annexConfig "alwayscompact") True
|
2020-03-02 19:50:40 +00:00
|
|
|
, annexCommitMessage = getmaybe (annexConfig "commitmessage")
|
|
|
|
|
, annexMergeAnnexBranches = getbool (annexConfig "merge-annex-branches") True
|
|
|
|
|
, annexDelayAdd = getmayberead (annexConfig "delayadd")
|
|
|
|
|
, annexHttpHeaders = getlist (annexConfig "http-headers")
|
|
|
|
|
, annexHttpHeadersCommand = getmaybe (annexConfig "http-headers-command")
|
2017-02-03 17:40:14 +00:00
|
|
|
, annexAutoCommit = configurable True $
|
2020-03-02 19:50:40 +00:00
|
|
|
getmaybebool (annexConfig "autocommit")
|
2017-06-01 16:46:36 +00:00
|
|
|
, annexResolveMerge = configurable True $
|
2020-03-02 19:50:40 +00:00
|
|
|
getmaybebool (annexConfig "resolvemerge")
|
git-annex pull and push
Split out two new commands, git-annex pull and git-annex push. Those plus a
git commit are equivilant to git-annex sync.
In a sense, git-annex sync conflates 3 things, and it would have been
better to have push and pull from the beginning and not sync. Although
note that git-annex sync --content is faster than a pull followed by a
push, because it only has to walk the tree once, look at preferred
content once, etc. So there is some value in git-annex sync in speed, as
well as user convenience.
And it would be hard to split out pull and push from sync, as far as the
implementaton goes. The implementation inside sync was easy, just adjust
SyncOptions so it does the right thing.
Note that the new commands default to syncing content, unless
annex.synccontent is explicitly set to false. I'd like sync to also do
that, but that's a hard transition to make. As a start to that
transition, I added a note to git-annex-sync.mdwn that it may start to
do so in a future version of git-annex. But a real transition would
necessarily involve displaying warnings when sync is used without
--content, and time.
Sponsored-by: Kevin Mueller on Patreon
2023-05-16 20:37:30 +00:00
|
|
|
, annexSyncContent = configurablemaybe $
|
2020-03-02 19:50:40 +00:00
|
|
|
getmaybebool (annexConfig "synccontent")
|
sync --only-annex and annex.synconlyannex
* Added sync --only-annex, which syncs the git-annex branch and annexed
content but leaves managing the other git branches up to you.
* Added annex.synconlyannex git config setting, which can also be set with
git-annex config to configure sync in all clones of the repo.
Use case is then the user has their own git workflow, and wants to use
git-annex without disrupting that, so they sync --only-annex to get the
git-annex stuff in sync in addition to their usual git workflow.
When annex.synconlyannex is set, --not-only-annex can be used to override
it.
It's not entirely clear what --only-annex --commit or --only-annex
--push should do, and I left that combination not documented because I
don't know if I might want to change the current behavior, which is that
such options do not override the --only-annex. My gut feeling is that
there is no good reasons to use such combinations; if you want to use
your own git workflow, you'll be doing your own committing and pulling
and pushing.
A subtle question is, how should import/export special remotes be handled?
Importing updates their remote tracking branch and merges it into master.
If --only-annex prevented that git branch stuff, then it would prevent
exporting to the special remote, in the case where it has changes that
were not imported yet, because there would be a unresolved conflict.
I decided that it's best to treat the fact that there's a remote tracking
branch for import/export as an implementation detail in this case. The more
important thing is that an import/export special remote is entirely annexed
content, and so it makes a lot of sense that --only-annex will still sync
with it.
2020-02-17 19:19:58 +00:00
|
|
|
, annexSyncOnlyAnnex = configurable False $
|
2020-03-02 19:50:40 +00:00
|
|
|
getmaybebool (annexConfig "synconlyannex")
|
|
|
|
|
, annexDebug = getbool (annexConfig "debug") False
|
2021-04-06 19:14:00 +00:00
|
|
|
, annexDebugFilter = getmaybe (annexConfig "debugfilter")
|
2020-03-02 19:50:40 +00:00
|
|
|
, annexWebOptions = getwords (annexConfig "web-options")
|
|
|
|
|
, annexYoutubeDlOptions = getwords (annexConfig "youtube-dl-options")
|
2021-08-27 13:44:23 +00:00
|
|
|
, annexYoutubeDlCommand = getmaybe (annexConfig "youtube-dl-command")
|
2020-03-02 19:50:40 +00:00
|
|
|
, annexAriaTorrentOptions = getwords (annexConfig "aria-torrent-options")
|
|
|
|
|
, annexCrippledFileSystem = getbool (annexConfig "crippledfilesystem") False
|
git-annex config annex.largefiles
annex.largefiles can be configured by git-annex config, to more easily set
a default that will also be used by clones, without needing to shoehorn the
expression into the gitattributes file. The git config and gitattributes
override that.
Whenever something is added to git-annex config, we have to consider what
happens if a user puts a purposfully bad value in there. Or, if a new
git-annex adds some new value that an old git-annex can't parse.
In this case, a global annex.largefiles that can't be parsed currently
makes an error be thrown. That might not be ideal, but the gitattribute
behaves the same, and is almost equally repo-global.
Performance notes:
git-annex add and addurl construct a matcher once
and uses it for every file, so the added time penalty for reading the global
config log is minor. If the gitattributes annex.largefiles were deprecated,
git-annex add would get around 2% faster (excluding hashing), because
looking that up for each file is not fast. So this new way of setting
it is progress toward speeding up add.
git-annex smudge does need to load the log every time. As well as checking
the git attribute. Not ideal. Setting annex.gitaddtoannex=false avoids
both overheads.
2019-12-20 16:12:31 +00:00
|
|
|
, annexLargeFiles = configurable Nothing $
|
2020-03-02 19:50:40 +00:00
|
|
|
fmap Just $ getmaybe (annexConfig "largefiles")
|
|
|
|
|
, annexDotFiles = configurable False $
|
|
|
|
|
getmaybebool (annexConfig "dotfiles")
|
|
|
|
|
, annexGitAddToAnnex = getbool (annexConfig "gitaddtoannex") True
|
|
|
|
|
, annexAddSmallFiles = getbool (annexConfig "addsmallfiles") True
|
|
|
|
|
, annexFsckNudge = getbool (annexConfig "fscknudge") True
|
|
|
|
|
, annexAutoUpgrade = toAutoUpgrade $
|
|
|
|
|
getmaybe (annexConfig "autoupgrade")
|
2020-08-15 19:53:35 +00:00
|
|
|
, annexExpireUnused = either (const Nothing) Just . parseDuration
|
2020-03-02 19:50:40 +00:00
|
|
|
<$> getmaybe (annexConfig "expireunused")
|
Added annex.freezecontent-command and annex.thawcontent-command configs
Freeze first sets the file perms, and then runs
freezecontent-command. Thaw runs thawcontent-command before
restoring file permissions. This is in case the freeze command
prevents changing file perms, as eg setting a file immutable does.
Also, changing file perms tends to mess up previously set ACLs.
git-annex init's probe for crippled filesystem uses them, so if file perms
don't work, but freezecontent-command manages to prevent write to a file,
it won't treat the filesystem as crippled.
When the the filesystem has been probed as crippled, the hooks are not
used, because there seems to be no point then; git-annex won't be relying
on locking annex objects down. Also, this avoids them being run when the
file perms have not been changed, in case they somehow rely on
git-annex's setting of the file perms in order to work.
Sponsored-by: Dartmouth College's Datalad project
2021-06-21 18:40:20 +00:00
|
|
|
, annexFreezeContentCommand = getmaybe (annexConfig "freezecontent-command")
|
|
|
|
|
, annexThawContentCommand = getmaybe (annexConfig "thawcontent-command")
|
2020-03-02 19:50:40 +00:00
|
|
|
, annexSecureEraseCommand = getmaybe (annexConfig "secure-erase-command")
|
|
|
|
|
, annexGenMetaData = getbool (annexConfig "genmetadata") False
|
|
|
|
|
, annexListen = getmaybe (annexConfig "listen")
|
|
|
|
|
, annexStartupScan = getbool (annexConfig "startupscan") True
|
|
|
|
|
, annexHardLink = getbool (annexConfig "hardlink") False
|
|
|
|
|
, annexThin = getbool (annexConfig "thin") False
|
2015-01-27 21:38:06 +00:00
|
|
|
, annexDifferences = getDifferences r
|
2015-05-14 19:44:08 +00:00
|
|
|
, annexUsedRefSpec = either (const Nothing) Just . parseRefSpec
|
2020-03-02 19:50:40 +00:00
|
|
|
=<< getmaybe (annexConfig "used-refspec")
|
|
|
|
|
, annexVerify = getbool (annexConfig "verify") True
|
|
|
|
|
, annexPidLock = getbool (annexConfig "pidlock") False
|
2015-11-12 21:47:31 +00:00
|
|
|
, annexPidLockTimeout = Seconds $ fromMaybe 300 $
|
2020-03-02 19:50:40 +00:00
|
|
|
getmayberead (annexConfig "pidlocktimeout")
|
2022-08-12 17:18:15 +00:00
|
|
|
, annexDbDir = (\d -> toRawFilePath d P.</> fromUUID hereuuid)
|
|
|
|
|
<$> getmaybe (annexConfig "dbdir")
|
2019-12-20 19:01:34 +00:00
|
|
|
, annexAddUnlocked = configurable Nothing $
|
2020-03-02 19:50:40 +00:00
|
|
|
fmap Just $ getmaybe (annexConfig "addunlocked")
|
|
|
|
|
, annexSecureHashesOnly = getbool (annexConfig "securehashesonly") False
|
|
|
|
|
, annexRetry = getmayberead (annexConfig "retry")
|
2020-09-04 19:16:40 +00:00
|
|
|
, annexForwardRetry = getmayberead (annexConfig "forward-retry")
|
2018-03-24 14:37:25 +00:00
|
|
|
, annexRetryDelay = Seconds
|
2020-03-02 19:50:40 +00:00
|
|
|
<$> getmayberead (annexConfig "retrydelay")
|
limit url downloads to whitelisted schemes
Security fix! Allowing any schemes, particularly file: and
possibly others like scp: allowed file exfiltration by anyone who had
write access to the git repository, since they could add an annexed file
using such an url, or using an url that redirected to such an url,
and wait for the victim to get it into their repository and send them a copy.
* Added annex.security.allowed-url-schemes setting, which defaults
to only allowing http and https URLs. Note especially that file:/
is no longer enabled by default.
* Removed annex.web-download-command, since its interface does not allow
supporting annex.security.allowed-url-schemes across redirects.
If you used this setting, you may want to instead use annex.web-options
to pass options to curl.
With annex.web-download-command removed, nearly all url accesses in
git-annex are made via Utility.Url via http-client or curl. http-client
only supports http and https, so no problem there.
(Disabling one and not the other is not implemented.)
Used curl --proto to limit the allowed url schemes.
Note that this will cause git annex fsck --from web to mark files using
a disallowed url scheme as not being present in the web. That seems
acceptable; fsck --from web also does that when a web server is not available.
youtube-dl already disabled file: itself (probably for similar
reasons). The scheme check was also added to youtube-dl urls for
completeness, although that check won't catch any redirects it might
follow. But youtube-dl goes off and does its own thing with other
protocols anyway, so that's fine.
Special remotes that support other domain-specific url schemes are not
affected by this change. In the bittorrent remote, aria2c can still
download magnet: links. The download of the .torrent file is
otherwise now limited by annex.security.allowed-url-schemes.
This does not address any external special remotes that might download
an url themselves. Current thinking is all external special remotes will
need to be audited for this problem, although many of them will use
http libraries that only support http and not curl's menagarie.
The related problem of accessing private localhost and LAN urls is not
addressed by this commit.
This commit was sponsored by Brett Eisenberg on Patreon.
2018-06-15 20:52:24 +00:00
|
|
|
, annexAllowedUrlSchemes = S.fromList $ map mkScheme $
|
2018-06-18 19:36:12 +00:00
|
|
|
maybe ["http", "https", "ftp"] words $
|
2020-03-02 19:50:40 +00:00
|
|
|
getmaybe (annexConfig "security.allowed-url-schemes")
|
2019-05-30 16:43:40 +00:00
|
|
|
, annexAllowedIPAddresses = fromMaybe "" $
|
2020-03-02 19:50:40 +00:00
|
|
|
getmaybe (annexConfig "security.allowed-ip-addresses")
|
2019-05-30 16:43:40 +00:00
|
|
|
<|>
|
2020-03-02 19:50:40 +00:00
|
|
|
getmaybe (annexConfig "security.allowed-http-addresses") -- old name
|
2018-06-21 17:34:11 +00:00
|
|
|
, annexAllowUnverifiedDownloads = (== Just "ACKTHPPT") $
|
2020-03-02 19:50:40 +00:00
|
|
|
getmaybe (annexConfig "security.allow-unverified-downloads")
|
|
|
|
|
, annexMaxExtensionLength = getmayberead (annexConfig "maxextensionlength")
|
2019-05-10 17:24:31 +00:00
|
|
|
, annexJobs = fromMaybe NonConcurrent $
|
2020-03-02 19:50:40 +00:00
|
|
|
parseConcurrency =<< getmaybe (annexConfig "jobs")
|
|
|
|
|
, annexCacheCreds = getbool (annexConfig "cachecreds") True
|
|
|
|
|
, annexAutoUpgradeRepository = getbool (annexConfig "autoupgraderepository") True
|
|
|
|
|
, annexCommitMode = if getbool (annexConfig "allowsign") False
|
2019-11-11 22:20:35 +00:00
|
|
|
then ManualCommit
|
|
|
|
|
else AutomaticCommit
|
2022-02-18 17:18:05 +00:00
|
|
|
, annexSkipUnknown = getbool (annexConfig "skipunknown") False
|
2020-11-16 18:09:55 +00:00
|
|
|
, annexAdjustedBranchRefresh = fromMaybe
|
|
|
|
|
-- parse as bool if it's not a number
|
|
|
|
|
(if getbool "adjustedbranchrefresh" False then 1 else 0)
|
|
|
|
|
(getmayberead (annexConfig "adjustedbranchrefresh"))
|
2021-03-23 18:04:34 +00:00
|
|
|
, annexSupportUnlocked = getbool (annexConfig "supportunlocked") True
|
2015-09-09 22:06:49 +00:00
|
|
|
, coreSymlinks = getbool "core.symlinks" True
|
|
|
|
|
, coreSharedRepository = getSharedRepository r
|
git style filename quoting controlled by core.quotePath
This is by no means complete, but escaping filenames in actionItemDesc does
cover most commands.
Note that for ActionItemBranchFilePath, the value is branch:file, and I
choose to only quote the file part (if necessary). I considered quoting the
whole thing. But, branch names cannot contain control characters, and while
they can contain unicode, git coes not quote unicode when displaying branch
names. So, it would be surprising for git-annex to quote unicode in a
branch name.
The find command is the most obvious command that still needs to be
dealt with. There are probably other places that filenames also get
displayed, eg embedded in error messages.
Some other commands use ActionItemOther with a filename, I think that
ActionItemOther should either be pre-sanitized, or should explicitly not
be used for filenames, so that needs more work.
When --json is used, unicode does not get escaped, but control
characters were already escaped in json.
(Key escaping may turn out to be needed, but I'm ignoring that for now.)
Sponsored-by: unqueued on Patreon
2023-04-08 18:20:02 +00:00
|
|
|
, coreQuotePath = QuotePath (getbool "core.quotepath" True)
|
2017-02-17 18:04:43 +00:00
|
|
|
, receiveDenyCurrentBranch = getDenyCurrentBranch r
|
2015-09-09 22:06:49 +00:00
|
|
|
, gcryptId = getmaybe "core.gcrypt-id"
|
|
|
|
|
, gpgCmd = mkGpgCmd (getmaybe "gpg.program")
|
2020-09-07 17:50:58 +00:00
|
|
|
, mergeDirectoryRenames = getmaybe "directoryrenames"
|
2021-04-23 18:21:57 +00:00
|
|
|
, annexPrivateRepos = S.fromList $ concat
|
|
|
|
|
[ if getbool (annexConfig "private") False
|
|
|
|
|
then [hereuuid]
|
|
|
|
|
else []
|
|
|
|
|
, let get (k, v)
|
|
|
|
|
| Git.Config.isTrueFalse' v /= Just True = Nothing
|
|
|
|
|
| isRemoteKey (remoteAnnexConfigEnd "private") k = do
|
|
|
|
|
remotename <- remoteKeyToRemoteName k
|
|
|
|
|
toUUID <$> Git.Config.getMaybe
|
|
|
|
|
(remoteAnnexConfig remotename "uuid") r
|
|
|
|
|
| otherwise = Nothing
|
|
|
|
|
in mapMaybe get (M.toList (Git.config r))
|
|
|
|
|
]
|
2022-09-30 14:56:17 +00:00
|
|
|
, annexAdviceNoSshCaching = getbool (annexConfig "advicenosshcaching") True
|
2023-02-07 20:28:46 +00:00
|
|
|
, annexViewUnsetDirectory = ViewUnset $ fromMaybe "_" $
|
|
|
|
|
getmaybe (annexConfig "viewunsetdirectory")
|
2013-01-01 17:52:47 +00:00
|
|
|
}
|
|
|
|
|
where
|
2015-01-28 20:11:28 +00:00
|
|
|
getbool k d = fromMaybe d $ getmaybebool k
|
2019-12-20 19:01:34 +00:00
|
|
|
getmaybebool k = Git.Config.isTrueFalse' =<< getmaybe' k
|
2013-01-01 17:52:47 +00:00
|
|
|
getmayberead k = readish =<< getmaybe k
|
2019-12-05 18:36:43 +00:00
|
|
|
getmaybe = fmap fromConfigValue . getmaybe'
|
2019-11-27 20:54:11 +00:00
|
|
|
getmaybe' k = Git.Config.getMaybe k r
|
2019-12-05 18:36:43 +00:00
|
|
|
getlist k = map fromConfigValue $ Git.Config.getList k r
|
2013-01-27 13:33:19 +00:00
|
|
|
getwords k = fromMaybe [] $ words <$> getmaybe k
|
2013-01-01 17:52:47 +00:00
|
|
|
|
2017-02-03 17:40:14 +00:00
|
|
|
configurable d Nothing = DefaultConfig d
|
git-annex config annex.largefiles
annex.largefiles can be configured by git-annex config, to more easily set
a default that will also be used by clones, without needing to shoehorn the
expression into the gitattributes file. The git config and gitattributes
override that.
Whenever something is added to git-annex config, we have to consider what
happens if a user puts a purposfully bad value in there. Or, if a new
git-annex adds some new value that an old git-annex can't parse.
In this case, a global annex.largefiles that can't be parsed currently
makes an error be thrown. That might not be ideal, but the gitattribute
behaves the same, and is almost equally repo-global.
Performance notes:
git-annex add and addurl construct a matcher once
and uses it for every file, so the added time penalty for reading the global
config log is minor. If the gitattributes annex.largefiles were deprecated,
git-annex add would get around 2% faster (excluding hashing), because
looking that up for each file is not fast. So this new way of setting
it is progress toward speeding up add.
git-annex smudge does need to load the log every time. As well as checking
the git attribute. Not ideal. Setting annex.gitaddtoannex=false avoids
both overheads.
2019-12-20 16:12:31 +00:00
|
|
|
configurable _ (Just v) = case configsource of
|
|
|
|
|
FromGitConfig -> HasGitConfig v
|
|
|
|
|
FromGlobalConfig -> HasGlobalConfig v
|
git-annex pull and push
Split out two new commands, git-annex pull and git-annex push. Those plus a
git commit are equivilant to git-annex sync.
In a sense, git-annex sync conflates 3 things, and it would have been
better to have push and pull from the beginning and not sync. Although
note that git-annex sync --content is faster than a pull followed by a
push, because it only has to walk the tree once, look at preferred
content once, etc. So there is some value in git-annex sync in speed, as
well as user convenience.
And it would be hard to split out pull and push from sync, as far as the
implementaton goes. The implementation inside sync was easy, just adjust
SyncOptions so it does the right thing.
Note that the new commands default to syncing content, unless
annex.synccontent is explicitly set to false. I'd like sync to also do
that, but that's a hard transition to make. As a start to that
transition, I added a note to git-annex-sync.mdwn that it may start to
do so in a future version of git-annex. But a real transition would
necessarily involve displaying warnings when sync is used without
--content, and time.
Sponsored-by: Kevin Mueller on Patreon
2023-05-16 20:37:30 +00:00
|
|
|
|
|
|
|
|
configurablemaybe Nothing = DefaultConfig Nothing
|
|
|
|
|
configurablemaybe (Just v) = case configsource of
|
|
|
|
|
FromGitConfig -> HasGitConfig (Just v)
|
|
|
|
|
FromGlobalConfig -> HasGlobalConfig (Just v)
|
2017-02-03 17:40:14 +00:00
|
|
|
|
2013-01-01 17:52:47 +00:00
|
|
|
onemegabyte = 1000000
|
2021-04-23 18:21:57 +00:00
|
|
|
|
|
|
|
|
hereuuid = maybe NoUUID toUUID $ getmaybe (annexConfig "uuid")
|
2013-01-01 17:52:47 +00:00
|
|
|
|
2017-02-03 17:40:14 +00:00
|
|
|
{- Merge a GitConfig that comes from git-config with one containing
|
|
|
|
|
- repository-global defaults. -}
|
|
|
|
|
mergeGitConfig :: GitConfig -> GitConfig -> GitConfig
|
|
|
|
|
mergeGitConfig gitconfig repoglobals = gitconfig
|
|
|
|
|
{ annexAutoCommit = merge annexAutoCommit
|
2017-02-03 18:31:17 +00:00
|
|
|
, annexSyncContent = merge annexSyncContent
|
sync --only-annex and annex.synconlyannex
* Added sync --only-annex, which syncs the git-annex branch and annexed
content but leaves managing the other git branches up to you.
* Added annex.synconlyannex git config setting, which can also be set with
git-annex config to configure sync in all clones of the repo.
Use case is then the user has their own git workflow, and wants to use
git-annex without disrupting that, so they sync --only-annex to get the
git-annex stuff in sync in addition to their usual git workflow.
When annex.synconlyannex is set, --not-only-annex can be used to override
it.
It's not entirely clear what --only-annex --commit or --only-annex
--push should do, and I left that combination not documented because I
don't know if I might want to change the current behavior, which is that
such options do not override the --only-annex. My gut feeling is that
there is no good reasons to use such combinations; if you want to use
your own git workflow, you'll be doing your own committing and pulling
and pushing.
A subtle question is, how should import/export special remotes be handled?
Importing updates their remote tracking branch and merges it into master.
If --only-annex prevented that git branch stuff, then it would prevent
exporting to the special remote, in the case where it has changes that
were not imported yet, because there would be a unresolved conflict.
I decided that it's best to treat the fact that there's a remote tracking
branch for import/export as an implementation detail in this case. The more
important thing is that an import/export special remote is entirely annexed
content, and so it makes a lot of sense that --only-annex will still sync
with it.
2020-02-17 19:19:58 +00:00
|
|
|
, annexSyncOnlyAnnex = merge annexSyncOnlyAnnex
|
2019-12-20 14:55:23 +00:00
|
|
|
, annexResolveMerge = merge annexResolveMerge
|
git-annex config annex.largefiles
annex.largefiles can be configured by git-annex config, to more easily set
a default that will also be used by clones, without needing to shoehorn the
expression into the gitattributes file. The git config and gitattributes
override that.
Whenever something is added to git-annex config, we have to consider what
happens if a user puts a purposfully bad value in there. Or, if a new
git-annex adds some new value that an old git-annex can't parse.
In this case, a global annex.largefiles that can't be parsed currently
makes an error be thrown. That might not be ideal, but the gitattribute
behaves the same, and is almost equally repo-global.
Performance notes:
git-annex add and addurl construct a matcher once
and uses it for every file, so the added time penalty for reading the global
config log is minor. If the gitattributes annex.largefiles were deprecated,
git-annex add would get around 2% faster (excluding hashing), because
looking that up for each file is not fast. So this new way of setting
it is progress toward speeding up add.
git-annex smudge does need to load the log every time. As well as checking
the git attribute. Not ideal. Setting annex.gitaddtoannex=false avoids
both overheads.
2019-12-20 16:12:31 +00:00
|
|
|
, annexLargeFiles = merge annexLargeFiles
|
2019-12-26 20:24:40 +00:00
|
|
|
, annexDotFiles = merge annexDotFiles
|
2019-12-20 19:01:34 +00:00
|
|
|
, annexAddUnlocked = merge annexAddUnlocked
|
2017-02-03 17:40:14 +00:00
|
|
|
}
|
|
|
|
|
where
|
|
|
|
|
merge f = case f gitconfig of
|
git-annex config annex.largefiles
annex.largefiles can be configured by git-annex config, to more easily set
a default that will also be used by clones, without needing to shoehorn the
expression into the gitattributes file. The git config and gitattributes
override that.
Whenever something is added to git-annex config, we have to consider what
happens if a user puts a purposfully bad value in there. Or, if a new
git-annex adds some new value that an old git-annex can't parse.
In this case, a global annex.largefiles that can't be parsed currently
makes an error be thrown. That might not be ideal, but the gitattribute
behaves the same, and is almost equally repo-global.
Performance notes:
git-annex add and addurl construct a matcher once
and uses it for every file, so the added time penalty for reading the global
config log is minor. If the gitattributes annex.largefiles were deprecated,
git-annex add would get around 2% faster (excluding hashing), because
looking that up for each file is not fast. So this new way of setting
it is progress toward speeding up add.
git-annex smudge does need to load the log every time. As well as checking
the git attribute. Not ideal. Setting annex.gitaddtoannex=false avoids
both overheads.
2019-12-20 16:12:31 +00:00
|
|
|
HasGitConfig v -> HasGitConfig v
|
2017-02-03 17:40:14 +00:00
|
|
|
DefaultConfig d -> case f repoglobals of
|
git-annex config annex.largefiles
annex.largefiles can be configured by git-annex config, to more easily set
a default that will also be used by clones, without needing to shoehorn the
expression into the gitattributes file. The git config and gitattributes
override that.
Whenever something is added to git-annex config, we have to consider what
happens if a user puts a purposfully bad value in there. Or, if a new
git-annex adds some new value that an old git-annex can't parse.
In this case, a global annex.largefiles that can't be parsed currently
makes an error be thrown. That might not be ideal, but the gitattribute
behaves the same, and is almost equally repo-global.
Performance notes:
git-annex add and addurl construct a matcher once
and uses it for every file, so the added time penalty for reading the global
config log is minor. If the gitattributes annex.largefiles were deprecated,
git-annex add would get around 2% faster (excluding hashing), because
looking that up for each file is not fast. So this new way of setting
it is progress toward speeding up add.
git-annex smudge does need to load the log every time. As well as checking
the git attribute. Not ideal. Setting annex.gitaddtoannex=false avoids
both overheads.
2019-12-20 16:12:31 +00:00
|
|
|
HasGlobalConfig v -> HasGlobalConfig v
|
|
|
|
|
_ -> HasGitConfig d
|
|
|
|
|
HasGlobalConfig v -> HasGlobalConfig v
|
2017-02-03 17:40:14 +00:00
|
|
|
|
2020-03-02 19:50:40 +00:00
|
|
|
{- Configs that can be set repository-global. -}
|
|
|
|
|
globalConfigs :: [ConfigKey]
|
|
|
|
|
globalConfigs =
|
2021-04-26 17:50:37 +00:00
|
|
|
[ annexConfig "largefiles"
|
2020-03-02 19:50:40 +00:00
|
|
|
, annexConfig "dotfiles"
|
|
|
|
|
, annexConfig "addunlocked"
|
2021-04-26 17:50:37 +00:00
|
|
|
, annexConfig "autocommit"
|
|
|
|
|
, annexConfig "resolvemerge"
|
|
|
|
|
, annexConfig "synccontent"
|
|
|
|
|
, annexConfig "synconlyannex"
|
|
|
|
|
, annexConfig "securehashesonly"
|
2020-03-02 19:50:40 +00:00
|
|
|
]
|
|
|
|
|
|
2013-01-01 17:52:47 +00:00
|
|
|
{- Per-remote git-annex settings. Each setting corresponds to a git-config
|
|
|
|
|
- key such as <remote>.annex-foo, or if that is not set, a default from
|
removal of the rest of remoteGitConfig
In keyUrls, the GitConfig is used only by annexLocations
to support configured Differences. Since such configurations affect all
clones of a repository, the local repo's GitConfig must have the same
information as the remote's GitConfig would have. So, used getGitConfig
to get the local GitConfig, which is cached and so available cheaply.
That actually fixed a bug noone had ever noticed: keyUrls is
used for remotes accessed over http. The full git config of such a
remote is normally not available, so the remoteGitConfig that keyUrls
used would not have the necessary information in it.
In copyFromRemoteCheap', it uses gitAnnexLocation,
which does need the GitConfig of the remote repo itself in order to
check if it's crippled, supports symlinks, etc. So, made the
State include that GitConfig, cached. The use of gitAnnexLocation is
within a (not $ Git.repoIsUrl repo) guard, so it's local, and so
its git config will always be read and available.
(Note that gitAnnexLocation in turn calls annexLocations, so the
Differences config it uses in this case comes from the remote repo's
GitConfig and not from the local repo's GitConfig. As explained above
this is ok since they must have the same value.)
Not very happy with this mess of different GitConfigs not type-safe and
some read only sometimes etc. Very hairy. Think I got it this change
right. Test suite passes..
This commit was sponsored by Ethan Aubin.
2018-06-05 18:23:34 +00:00
|
|
|
- annex.foo.
|
|
|
|
|
-
|
|
|
|
|
- Note that this is from the perspective of the local repository,
|
|
|
|
|
- it is not influenced in any way by the contents of the remote
|
|
|
|
|
- repository's git config.
|
|
|
|
|
-}
|
2013-01-01 17:52:47 +00:00
|
|
|
data RemoteGitConfig = RemoteGitConfig
|
2017-08-17 18:04:29 +00:00
|
|
|
{ remoteAnnexCost :: DynamicConfig (Maybe Cost)
|
2017-08-17 16:26:14 +00:00
|
|
|
, remoteAnnexIgnore :: DynamicConfig Bool
|
|
|
|
|
, remoteAnnexSync :: DynamicConfig Bool
|
Added remote.<name>.annex-push and remote.<name>.annex-pull
The former can be useful to make remotes that don't get fully synced with
local changes, which comes up in a lot of situations.
The latter was mostly added for symmetry, but could be useful (though less
likely to be).
Implementing `remote.<name>.annex-pull` was a bit tricky, as there's no one
place where git-annex pulls/fetches from remotes. I audited all
instances of "fetch" and "pull". A few cases were left not checking this
config:
* Git.Repair can try to pull missing refs from a remote, and if the local
repo is corrupted, that seems a reasonable thing to do even though
the config would normally prevent it.
* Assistant.WebApp.Gpg and Remote.Gcrypt and Remote.Git do fetches
as part of the setup process of a remote. The config would probably not
be set then, and having the setup fail seems worse than honoring it if it
is already set.
I have not prevented all the code that does a "merge" from merging branches
from remotes with remote.<name>.annex-pull=false. That could perhaps
be done, but it would need a way to map from branch name to remote name,
and the way refspecs work makes that hard to get really correct. So if the
user fetches manually, the git-annex branch will get merged, for example.
Anther way of looking at/justifying this is that the setting is called
"annex-pull", not "annex-merge".
This commit was supported by the NSF-funded DataLad project.
2017-04-05 17:04:02 +00:00
|
|
|
, remoteAnnexPull :: Bool
|
|
|
|
|
, remoteAnnexPush :: Bool
|
2014-01-02 17:12:32 +00:00
|
|
|
, remoteAnnexReadOnly :: Bool
|
Do verification of checksums of annex objects downloaded from remotes.
* When annex objects are received into git repositories, their checksums are
verified then too.
* To get the old, faster, behavior of not verifying checksums, set
annex.verify=false, or remote.<name>.annex-verify=false.
* setkey, rekey: These commands also now verify that the provided file
matches the key, unless annex.verify=false.
* reinject: Already verified content; this can now be disabled by
setting annex.verify=false.
recvkey and reinject already did verification, so removed now duplicate
code from them. fsck still does its own verification, which is ok since it
does not use getViaTmp, so verification doesn't happen twice when using fsck
--from.
2015-10-01 19:54:37 +00:00
|
|
|
, remoteAnnexVerify :: Bool
|
2018-01-10 18:21:18 +00:00
|
|
|
, remoteAnnexCheckUUID :: Bool
|
2019-02-23 19:48:25 +00:00
|
|
|
, remoteAnnexTrackingBranch :: Maybe Git.Ref
|
2013-01-01 17:52:47 +00:00
|
|
|
, remoteAnnexTrustLevel :: Maybe String
|
|
|
|
|
, remoteAnnexStartCommand :: Maybe String
|
|
|
|
|
, remoteAnnexStopCommand :: Maybe String
|
2018-08-01 18:22:52 +00:00
|
|
|
, remoteAnnexSpeculatePresent :: Bool
|
2014-01-26 17:03:25 +00:00
|
|
|
, remoteAnnexBare :: Maybe Bool
|
2018-03-24 14:37:25 +00:00
|
|
|
, remoteAnnexRetry :: Maybe Integer
|
2020-09-04 19:16:40 +00:00
|
|
|
, remoteAnnexForwardRetry :: Maybe Integer
|
2018-03-24 14:37:25 +00:00
|
|
|
, remoteAnnexRetryDelay :: Maybe Seconds
|
2020-12-08 19:22:18 +00:00
|
|
|
, remoteAnnexStallDetection :: Maybe StallDetection
|
bwlimit
Added annex.bwlimit and remote.name.annex-bwlimit config that works for git
remotes and many but not all special remotes.
This nearly works, at least for a git remote on the same disk. With it set
to 100kb/1s, the meter displays an actual bandwidth of 128 kb/s, with
occasional spikes to 160 kb/s. So it needs to delay just a bit longer...
I'm unsure why.
However, at the beginning a lot of data flows before it determines the
right bandwidth limit. A granularity of less than 1s would probably improve
that.
And, I don't know yet if it makes sense to have it be 100ks/1s rather than
100kb/s. Is there a situation where the user would want a larger
granularity? Does granulatity need to be configurable at all? I only used that
format for the config really in order to reuse an existing parser.
This can't support for external special remotes, or for ones that
themselves shell out to an external command. (Well, it could, but it
would involve pausing and resuming the child process tree, which seems
very hard to implement and very strange besides.) There could also be some
built-in special remotes that it still doesn't work for, due to them not
having a progress meter whose displays blocks the bandwidth using thread.
But I don't think there are actually any that run a separate thread for
downloads than the thread that displays the progress meter.
Sponsored-by: Graham Spencer on Patreon
2021-09-21 20:58:02 +00:00
|
|
|
, remoteAnnexBwLimit :: Maybe BwRate
|
2018-09-25 19:32:50 +00:00
|
|
|
, remoteAnnexAllowUnverifiedDownloads :: Bool
|
2019-10-10 16:12:22 +00:00
|
|
|
, remoteAnnexConfigUUID :: Maybe UUID
|
2013-01-01 17:52:47 +00:00
|
|
|
|
2013-04-04 19:46:33 +00:00
|
|
|
{- These settings are specific to particular types of remotes
|
|
|
|
|
- including special remotes. -}
|
2014-05-16 11:34:43 +00:00
|
|
|
, remoteAnnexShell :: Maybe String
|
2013-01-01 17:52:47 +00:00
|
|
|
, remoteAnnexSshOptions :: [String]
|
|
|
|
|
, remoteAnnexRsyncOptions :: [String]
|
2014-02-02 20:06:34 +00:00
|
|
|
, remoteAnnexRsyncUploadOptions :: [String]
|
|
|
|
|
, remoteAnnexRsyncDownloadOptions :: [String]
|
2013-04-13 22:10:49 +00:00
|
|
|
, remoteAnnexRsyncTransport :: [String]
|
2013-03-11 01:33:13 +00:00
|
|
|
, remoteAnnexGnupgOptions :: [String]
|
2016-05-10 17:03:56 +00:00
|
|
|
, remoteAnnexGnupgDecryptOptions :: [String]
|
2013-01-01 17:52:47 +00:00
|
|
|
, remoteAnnexRsyncUrl :: Maybe String
|
|
|
|
|
, remoteAnnexBupRepo :: Maybe String
|
2020-12-18 20:03:51 +00:00
|
|
|
, remoteAnnexBorgRepo :: Maybe String
|
2014-01-08 20:14:37 +00:00
|
|
|
, remoteAnnexTahoe :: Maybe FilePath
|
2013-01-01 17:52:47 +00:00
|
|
|
, remoteAnnexBupSplitOptions :: [String]
|
|
|
|
|
, remoteAnnexDirectory :: Maybe FilePath
|
2018-03-27 16:41:57 +00:00
|
|
|
, remoteAnnexAndroidDirectory :: Maybe FilePath
|
|
|
|
|
, remoteAnnexAndroidSerial :: Maybe String
|
2013-09-24 21:25:47 +00:00
|
|
|
, remoteAnnexGCrypt :: Maybe String
|
2019-08-01 19:11:45 +00:00
|
|
|
, remoteAnnexGitLFS :: Bool
|
2014-05-15 18:44:00 +00:00
|
|
|
, remoteAnnexDdarRepo :: Maybe String
|
2013-01-01 17:52:47 +00:00
|
|
|
, remoteAnnexHookType :: Maybe String
|
external special remotes mostly implemented (untested)
This has not been tested at all. It compiles!
The only known missing things are support for encryption, and for get/set
of special remote configuration, and of key state. (The latter needs
separate work to add a new per-key log file to store that state.)
Only thing I don't much like is that initremote needs to be passed both
type=external and externaltype=foo. It would be better to have just
type=foo
Most of this is quite straightforward code, that largely wrote itself given
the types. The only tricky parts were:
* Need to lock the remote when using it to eg make a request, because
in theory git-annex could have multiple threads that each try to use
a remote at the same time. I don't think that git-annex ever does
that currently, but better safe than sorry.
* Rather than starting up every external special remote program when
git-annex starts, they are started only on demand, when first used.
This will avoid slowdown, especially when running fast git-annex query
commands. Once started, they keep running until git-annex stops, currently,
which may not be ideal, but it's hard to know a better time to stop them.
* Bit of a chicken and egg problem with caching the cost of the remote,
because setting annex-cost in the git config needs the remote to already
be set up. Managed to finesse that.
This commit was sponsored by Lukas Anzinger.
2013-12-26 22:23:13 +00:00
|
|
|
, remoteAnnexExternalType :: Maybe String
|
2013-01-01 17:52:47 +00:00
|
|
|
}
|
|
|
|
|
|
removal of the rest of remoteGitConfig
In keyUrls, the GitConfig is used only by annexLocations
to support configured Differences. Since such configurations affect all
clones of a repository, the local repo's GitConfig must have the same
information as the remote's GitConfig would have. So, used getGitConfig
to get the local GitConfig, which is cached and so available cheaply.
That actually fixed a bug noone had ever noticed: keyUrls is
used for remotes accessed over http. The full git config of such a
remote is normally not available, so the remoteGitConfig that keyUrls
used would not have the necessary information in it.
In copyFromRemoteCheap', it uses gitAnnexLocation,
which does need the GitConfig of the remote repo itself in order to
check if it's crippled, supports symlinks, etc. So, made the
State include that GitConfig, cached. The use of gitAnnexLocation is
within a (not $ Git.repoIsUrl repo) guard, so it's local, and so
its git config will always be read and available.
(Note that gitAnnexLocation in turn calls annexLocations, so the
Differences config it uses in this case comes from the remote repo's
GitConfig and not from the local repo's GitConfig. As explained above
this is ok since they must have the same value.)
Not very happy with this mess of different GitConfigs not type-safe and
some read only sometimes etc. Very hairy. Think I got it this change
right. Test suite passes..
This commit was sponsored by Ethan Aubin.
2018-06-05 18:23:34 +00:00
|
|
|
{- The Git.Repo is the local repository, which has the remote with the
|
|
|
|
|
- given RemoteName. -}
|
|
|
|
|
extractRemoteGitConfig :: Git.Repo -> RemoteName -> STM RemoteGitConfig
|
2017-08-17 16:26:14 +00:00
|
|
|
extractRemoteGitConfig r remotename = do
|
2017-08-17 18:04:29 +00:00
|
|
|
annexcost <- mkDynamicConfig readCommandRunner
|
|
|
|
|
(notempty $ getmaybe "cost-command")
|
|
|
|
|
(getmayberead "cost")
|
2017-08-17 16:26:14 +00:00
|
|
|
annexignore <- mkDynamicConfig unsuccessfullCommandRunner
|
|
|
|
|
(notempty $ getmaybe "ignore-command")
|
|
|
|
|
(getbool "ignore" False)
|
|
|
|
|
annexsync <- mkDynamicConfig successfullCommandRunner
|
|
|
|
|
(notempty $ getmaybe "sync-command")
|
|
|
|
|
(getbool "sync" True)
|
|
|
|
|
return $ RemoteGitConfig
|
2017-08-17 18:04:29 +00:00
|
|
|
{ remoteAnnexCost = annexcost
|
2017-08-17 16:26:14 +00:00
|
|
|
, remoteAnnexIgnore = annexignore
|
|
|
|
|
, remoteAnnexSync = annexsync
|
|
|
|
|
, remoteAnnexPull = getbool "pull" True
|
|
|
|
|
, remoteAnnexPush = getbool "push" True
|
|
|
|
|
, remoteAnnexReadOnly = getbool "readonly" False
|
2018-01-10 18:21:18 +00:00
|
|
|
, remoteAnnexCheckUUID = getbool "checkuuid" True
|
2017-08-17 16:26:14 +00:00
|
|
|
, remoteAnnexVerify = getbool "verify" True
|
2020-04-07 17:27:11 +00:00
|
|
|
, remoteAnnexTrackingBranch = Git.Ref . encodeBS <$>
|
2019-03-11 17:56:59 +00:00
|
|
|
( notempty (getmaybe "tracking-branch")
|
2019-02-23 19:48:25 +00:00
|
|
|
<|> notempty (getmaybe "export-tracking") -- old name
|
|
|
|
|
)
|
2017-08-17 16:26:14 +00:00
|
|
|
, remoteAnnexTrustLevel = notempty $ getmaybe "trustlevel"
|
|
|
|
|
, remoteAnnexStartCommand = notempty $ getmaybe "start-command"
|
|
|
|
|
, remoteAnnexStopCommand = notempty $ getmaybe "stop-command"
|
2018-08-01 18:22:52 +00:00
|
|
|
, remoteAnnexSpeculatePresent = getbool "speculate-present" False
|
2017-08-17 16:26:14 +00:00
|
|
|
, remoteAnnexBare = getmaybebool "bare"
|
2018-03-24 14:37:25 +00:00
|
|
|
, remoteAnnexRetry = getmayberead "retry"
|
2020-09-04 19:16:40 +00:00
|
|
|
, remoteAnnexForwardRetry = getmayberead "forward-retry"
|
2018-03-24 14:37:25 +00:00
|
|
|
, remoteAnnexRetryDelay = Seconds
|
|
|
|
|
<$> getmayberead "retrydelay"
|
2020-12-08 19:22:18 +00:00
|
|
|
, remoteAnnexStallDetection =
|
bwlimit
Added annex.bwlimit and remote.name.annex-bwlimit config that works for git
remotes and many but not all special remotes.
This nearly works, at least for a git remote on the same disk. With it set
to 100kb/1s, the meter displays an actual bandwidth of 128 kb/s, with
occasional spikes to 160 kb/s. So it needs to delay just a bit longer...
I'm unsure why.
However, at the beginning a lot of data flows before it determines the
right bandwidth limit. A granularity of less than 1s would probably improve
that.
And, I don't know yet if it makes sense to have it be 100ks/1s rather than
100kb/s. Is there a situation where the user would want a larger
granularity? Does granulatity need to be configurable at all? I only used that
format for the config really in order to reuse an existing parser.
This can't support for external special remotes, or for ones that
themselves shell out to an external command. (Well, it could, but it
would involve pausing and resuming the child process tree, which seems
very hard to implement and very strange besides.) There could also be some
built-in special remotes that it still doesn't work for, due to them not
having a progress meter whose displays blocks the bandwidth using thread.
But I don't think there are actually any that run a separate thread for
downloads than the thread that displays the progress meter.
Sponsored-by: Graham Spencer on Patreon
2021-09-21 20:58:02 +00:00
|
|
|
either (const Nothing) Just . parseStallDetection
|
2020-12-08 19:22:18 +00:00
|
|
|
=<< getmaybe "stalldetection"
|
2021-09-22 19:14:28 +00:00
|
|
|
, remoteAnnexBwLimit = do
|
|
|
|
|
sz <- readSize dataUnits =<< getmaybe "bwlimit"
|
|
|
|
|
return (BwRate sz (Duration 1))
|
2018-09-25 19:32:50 +00:00
|
|
|
, remoteAnnexAllowUnverifiedDownloads = (== Just "ACKTHPPT") $
|
|
|
|
|
getmaybe ("security-allow-unverified-downloads")
|
2019-10-10 16:12:22 +00:00
|
|
|
, remoteAnnexConfigUUID = toUUID <$> getmaybe "config-uuid"
|
2017-08-17 16:26:14 +00:00
|
|
|
, remoteAnnexShell = getmaybe "shell"
|
|
|
|
|
, remoteAnnexSshOptions = getoptions "ssh-options"
|
|
|
|
|
, remoteAnnexRsyncOptions = getoptions "rsync-options"
|
|
|
|
|
, remoteAnnexRsyncDownloadOptions = getoptions "rsync-download-options"
|
|
|
|
|
, remoteAnnexRsyncUploadOptions = getoptions "rsync-upload-options"
|
|
|
|
|
, remoteAnnexRsyncTransport = getoptions "rsync-transport"
|
|
|
|
|
, remoteAnnexGnupgOptions = getoptions "gnupg-options"
|
|
|
|
|
, remoteAnnexGnupgDecryptOptions = getoptions "gnupg-decrypt-options"
|
|
|
|
|
, remoteAnnexRsyncUrl = notempty $ getmaybe "rsyncurl"
|
|
|
|
|
, remoteAnnexBupRepo = getmaybe "buprepo"
|
2020-12-18 20:03:51 +00:00
|
|
|
, remoteAnnexBorgRepo = getmaybe "borgrepo"
|
2017-08-17 16:26:14 +00:00
|
|
|
, remoteAnnexTahoe = getmaybe "tahoe"
|
|
|
|
|
, remoteAnnexBupSplitOptions = getoptions "bup-split-options"
|
|
|
|
|
, remoteAnnexDirectory = notempty $ getmaybe "directory"
|
2018-03-27 16:41:57 +00:00
|
|
|
, remoteAnnexAndroidDirectory = notempty $ getmaybe "androiddirectory"
|
|
|
|
|
, remoteAnnexAndroidSerial = notempty $ getmaybe "androidserial"
|
2017-08-17 16:26:14 +00:00
|
|
|
, remoteAnnexGCrypt = notempty $ getmaybe "gcrypt"
|
2019-08-01 19:11:45 +00:00
|
|
|
, remoteAnnexGitLFS = getbool "git-lfs" False
|
2017-08-17 16:26:14 +00:00
|
|
|
, remoteAnnexDdarRepo = getmaybe "ddarrepo"
|
|
|
|
|
, remoteAnnexHookType = notempty $ getmaybe "hooktype"
|
|
|
|
|
, remoteAnnexExternalType = notempty $ getmaybe "externaltype"
|
|
|
|
|
}
|
2013-01-01 17:52:47 +00:00
|
|
|
where
|
2015-01-28 20:11:28 +00:00
|
|
|
getbool k d = fromMaybe d $ getmaybebool k
|
2019-12-20 19:01:34 +00:00
|
|
|
getmaybebool k = Git.Config.isTrueFalse' =<< getmaybe' k
|
2013-01-01 17:52:47 +00:00
|
|
|
getmayberead k = readish =<< getmaybe k
|
2019-12-05 18:36:43 +00:00
|
|
|
getmaybe = fmap fromConfigValue . getmaybe'
|
2021-09-22 14:41:39 +00:00
|
|
|
getmaybe' k =
|
|
|
|
|
Git.Config.getMaybe (remoteAnnexConfig remotename k) r
|
|
|
|
|
<|>
|
|
|
|
|
Git.Config.getMaybe (annexConfig k) r
|
2013-01-01 17:52:47 +00:00
|
|
|
getoptions k = fromMaybe [] $ words <$> getmaybe k
|
|
|
|
|
|
|
|
|
|
notempty :: Maybe String -> Maybe String
|
|
|
|
|
notempty Nothing = Nothing
|
|
|
|
|
notempty (Just "") = Nothing
|
|
|
|
|
notempty (Just s) = Just s
|
|
|
|
|
|
2017-08-17 16:26:14 +00:00
|
|
|
dummyRemoteGitConfig :: IO RemoteGitConfig
|
|
|
|
|
dummyRemoteGitConfig = atomically $
|
|
|
|
|
extractRemoteGitConfig Git.Construct.fromUnknown "dummy"
|
2020-03-02 19:50:40 +00:00
|
|
|
|
|
|
|
|
type UnqualifiedConfigKey = B.ByteString
|
|
|
|
|
|
|
|
|
|
{- A global annex setting in git config. -}
|
|
|
|
|
annexConfig :: UnqualifiedConfigKey -> ConfigKey
|
|
|
|
|
annexConfig key = ConfigKey ("annex." <> key)
|
|
|
|
|
|
|
|
|
|
class RemoteNameable r where
|
|
|
|
|
getRemoteName :: r -> RemoteName
|
|
|
|
|
|
|
|
|
|
instance RemoteNameable Git.Repo where
|
|
|
|
|
getRemoteName r = fromMaybe "" (Git.remoteName r)
|
|
|
|
|
|
|
|
|
|
instance RemoteNameable RemoteName where
|
|
|
|
|
getRemoteName = id
|
|
|
|
|
|
|
|
|
|
{- A per-remote annex setting in git config. -}
|
|
|
|
|
remoteAnnexConfig :: RemoteNameable r => r -> UnqualifiedConfigKey -> ConfigKey
|
2021-04-23 18:21:57 +00:00
|
|
|
remoteAnnexConfig r = remoteConfig r . remoteAnnexConfigEnd
|
|
|
|
|
|
|
|
|
|
remoteAnnexConfigEnd :: UnqualifiedConfigKey -> UnqualifiedConfigKey
|
|
|
|
|
remoteAnnexConfigEnd key = "annex-" <> key
|
2020-03-02 19:50:40 +00:00
|
|
|
|
|
|
|
|
{- A per-remote setting in git config. -}
|
|
|
|
|
remoteConfig :: RemoteNameable r => r -> UnqualifiedConfigKey -> ConfigKey
|
|
|
|
|
remoteConfig r key = ConfigKey $
|
2021-08-11 00:45:02 +00:00
|
|
|
"remote." <> encodeBS (getRemoteName r) <> "." <> key
|