2013-01-01 17:52:47 +00:00
|
|
|
{- git-annex configuration
|
|
|
|
-
|
assistant unused file handling
Make sanity checker run git annex unused daily, and queue up transfers
of unused files to any remotes that will have them. The transfer retrying
code works for us here, so eg when a backup disk remote is plugged in,
any transfers to it are done. Once the unused files reach a remote,
they'll be removed locally as unwanted.
If the setup does not cause unused files to go to a remote, they'll pile
up, and the sanity checker detects this using some heuristics that are
pretty good -- 1000 unused files, or 10% of disk used by unused files,
or more disk wasted by unused files than is left free. Once it detects
this, it pops up an alert in the webapp, with a button to take action.
TODO: Webapp UI to configure this, and also the ability to launch an
immediate cleanup of all unused files.
This commit was sponsored by Simon Michael.
2014-01-23 02:48:56 +00:00
|
|
|
- Copyright 2012-2014 Joey Hess <joey@kitenet.net>
|
2013-01-01 17:52:47 +00:00
|
|
|
-
|
|
|
|
- Licensed under the GNU GPL version 3 or higher.
|
|
|
|
-}
|
|
|
|
|
|
|
|
module Types.GitConfig (
|
|
|
|
GitConfig(..),
|
|
|
|
extractGitConfig,
|
|
|
|
RemoteGitConfig(..),
|
|
|
|
extractRemoteGitConfig,
|
|
|
|
) where
|
|
|
|
|
|
|
|
import Common
|
|
|
|
import qualified Git
|
|
|
|
import qualified Git.Config
|
|
|
|
import Utility.DataUnits
|
2013-03-13 20:16:01 +00:00
|
|
|
import Config.Cost
|
2013-11-22 20:04:20 +00:00
|
|
|
import Types.Distribution
|
2014-01-13 18:41:10 +00:00
|
|
|
import Types.Availability
|
2014-01-21 21:08:49 +00:00
|
|
|
import Types.NumCopies
|
assistant unused file handling
Make sanity checker run git annex unused daily, and queue up transfers
of unused files to any remotes that will have them. The transfer retrying
code works for us here, so eg when a backup disk remote is plugged in,
any transfers to it are done. Once the unused files reach a remote,
they'll be removed locally as unwanted.
If the setup does not cause unused files to go to a remote, they'll pile
up, and the sanity checker detects this using some heuristics that are
pretty good -- 1000 unused files, or 10% of disk used by unused files,
or more disk wasted by unused files than is left free. Once it detects
this, it pops up an alert in the webapp, with a button to take action.
TODO: Webapp UI to configure this, and also the ability to launch an
immediate cleanup of all unused files.
This commit was sponsored by Simon Michael.
2014-01-23 02:48:56 +00:00
|
|
|
import Utility.HumanTime
|
2013-01-01 17:52:47 +00:00
|
|
|
|
|
|
|
{- Main git-annex settings. Each setting corresponds to a git-config key
|
|
|
|
- such as annex.foo -}
|
|
|
|
data GitConfig = GitConfig
|
|
|
|
{ annexVersion :: Maybe String
|
2014-01-21 21:08:49 +00:00
|
|
|
, annexNumCopies :: Maybe NumCopies
|
2013-01-01 17:52:47 +00:00
|
|
|
, annexDiskReserve :: Integer
|
|
|
|
, annexDirect :: Bool
|
|
|
|
, annexBackends :: [String]
|
|
|
|
, annexQueueSize :: Maybe Int
|
|
|
|
, annexBloomCapacity :: Maybe Int
|
|
|
|
, annexBloomAccuracy :: Maybe Int
|
|
|
|
, annexSshCaching :: Maybe Bool
|
|
|
|
, annexAlwaysCommit :: Bool
|
|
|
|
, annexDelayAdd :: Maybe Int
|
|
|
|
, annexHttpHeaders :: [String]
|
|
|
|
, annexHttpHeadersCommand :: Maybe String
|
2013-01-27 11:43:05 +00:00
|
|
|
, annexAutoCommit :: Bool
|
2013-06-18 00:41:17 +00:00
|
|
|
, annexDebug :: Bool
|
2013-01-27 13:33:19 +00:00
|
|
|
, annexWebOptions :: [String]
|
2013-08-22 22:25:21 +00:00
|
|
|
, annexQuviOptions :: [String]
|
2014-12-17 03:22:46 +00:00
|
|
|
, annexAriaTorrentOptions :: [String]
|
2013-04-09 03:34:05 +00:00
|
|
|
, annexWebDownloadCommand :: Maybe String
|
2013-02-14 18:10:36 +00:00
|
|
|
, annexCrippledFileSystem :: Bool
|
2013-03-29 20:17:13 +00:00
|
|
|
, annexLargeFiles :: Maybe String
|
2013-10-29 20:48:06 +00:00
|
|
|
, annexFsckNudge :: Bool
|
2013-11-22 20:04:20 +00:00
|
|
|
, annexAutoUpgrade :: AutoUpgrade
|
2014-01-23 03:10:51 +00:00
|
|
|
, annexExpireUnused :: Maybe (Maybe Duration)
|
2014-01-24 16:58:52 +00:00
|
|
|
, annexSecureEraseCommand :: Maybe String
|
2014-02-23 04:08:29 +00:00
|
|
|
, annexGenMetaData :: Bool
|
2014-03-01 04:31:17 +00:00
|
|
|
, annexListen :: Maybe String
|
2014-03-05 21:44:14 +00:00
|
|
|
, annexStartupScan :: Bool
|
2014-09-05 17:44:09 +00:00
|
|
|
, annexHardLink :: Bool
|
2013-02-15 20:02:35 +00:00
|
|
|
, coreSymlinks :: Bool
|
2013-09-24 21:25:47 +00:00
|
|
|
, gcryptId :: Maybe String
|
2013-01-01 17:52:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
extractGitConfig :: Git.Repo -> GitConfig
|
|
|
|
extractGitConfig r = GitConfig
|
2013-02-15 20:02:35 +00:00
|
|
|
{ annexVersion = notempty $ getmaybe (annex "version")
|
2014-01-21 21:08:49 +00:00
|
|
|
, annexNumCopies = NumCopies <$> getmayberead (annex "numcopies")
|
2013-01-01 17:52:47 +00:00
|
|
|
, annexDiskReserve = fromMaybe onemegabyte $
|
2013-02-15 20:02:35 +00:00
|
|
|
readSize dataUnits =<< getmaybe (annex "diskreserve")
|
|
|
|
, annexDirect = getbool (annex "direct") False
|
|
|
|
, annexBackends = getwords (annex "backends")
|
|
|
|
, annexQueueSize = getmayberead (annex "queuesize")
|
|
|
|
, annexBloomCapacity = getmayberead (annex "bloomcapacity")
|
|
|
|
, annexBloomAccuracy = getmayberead (annex "bloomaccuracy")
|
|
|
|
, annexSshCaching = getmaybebool (annex "sshcaching")
|
|
|
|
, annexAlwaysCommit = getbool (annex "alwayscommit") True
|
|
|
|
, annexDelayAdd = getmayberead (annex "delayadd")
|
|
|
|
, annexHttpHeaders = getlist (annex "http-headers")
|
|
|
|
, annexHttpHeadersCommand = getmaybe (annex "http-headers-command")
|
|
|
|
, annexAutoCommit = getbool (annex "autocommit") True
|
2013-06-18 00:41:17 +00:00
|
|
|
, annexDebug = getbool (annex "debug") False
|
2013-02-15 20:02:35 +00:00
|
|
|
, annexWebOptions = getwords (annex "web-options")
|
2013-08-22 22:25:21 +00:00
|
|
|
, annexQuviOptions = getwords (annex "quvi-options")
|
2014-12-17 03:22:46 +00:00
|
|
|
, annexAriaTorrentOptions = getwords (annex "aria-torrent-options")
|
2013-04-09 03:34:05 +00:00
|
|
|
, annexWebDownloadCommand = getmaybe (annex "web-download-command")
|
2013-02-15 20:02:35 +00:00
|
|
|
, annexCrippledFileSystem = getbool (annex "crippledfilesystem") False
|
2013-03-29 20:17:13 +00:00
|
|
|
, annexLargeFiles = getmaybe (annex "largefiles")
|
2013-10-29 20:48:06 +00:00
|
|
|
, annexFsckNudge = getbool (annex "fscknudge") True
|
2013-11-22 20:04:20 +00:00
|
|
|
, annexAutoUpgrade = toAutoUpgrade $ getmaybe (annex "autoupgrade")
|
2014-01-23 03:10:51 +00:00
|
|
|
, annexExpireUnused = maybe Nothing Just . parseDuration
|
|
|
|
<$> getmaybe (annex "expireunused")
|
2014-01-24 16:58:52 +00:00
|
|
|
, annexSecureEraseCommand = getmaybe (annex "secure-erase-command")
|
2014-02-23 04:08:29 +00:00
|
|
|
, annexGenMetaData = getbool (annex "genmetadata") False
|
2014-03-01 04:31:17 +00:00
|
|
|
, annexListen = getmaybe (annex "listen")
|
2014-03-05 21:44:14 +00:00
|
|
|
, annexStartupScan = getbool (annex "startupscan") True
|
2014-09-05 17:44:09 +00:00
|
|
|
, annexHardLink = getbool (annex "hardlink") False
|
2013-02-15 20:02:35 +00:00
|
|
|
, coreSymlinks = getbool "core.symlinks" True
|
2013-09-24 21:25:47 +00:00
|
|
|
, gcryptId = getmaybe "core.gcrypt-id"
|
2013-01-01 17:52:47 +00:00
|
|
|
}
|
|
|
|
where
|
|
|
|
getbool k def = fromMaybe def $ getmaybebool k
|
|
|
|
getmaybebool k = Git.Config.isTrue =<< getmaybe k
|
|
|
|
getmayberead k = readish =<< getmaybe k
|
2013-02-15 20:02:35 +00:00
|
|
|
getmaybe k = Git.Config.getMaybe k r
|
|
|
|
getlist k = Git.Config.getList k r
|
2013-01-27 13:33:19 +00:00
|
|
|
getwords k = fromMaybe [] $ words <$> getmaybe k
|
2013-01-01 17:52:47 +00:00
|
|
|
|
2013-02-15 20:02:35 +00:00
|
|
|
annex k = "annex." ++ k
|
2013-01-01 17:52:47 +00:00
|
|
|
|
|
|
|
onemegabyte = 1000000
|
|
|
|
|
|
|
|
{- Per-remote git-annex settings. Each setting corresponds to a git-config
|
|
|
|
- key such as <remote>.annex-foo, or if that is not set, a default from
|
|
|
|
- annex.foo -}
|
|
|
|
data RemoteGitConfig = RemoteGitConfig
|
2013-03-13 20:16:01 +00:00
|
|
|
{ remoteAnnexCost :: Maybe Cost
|
2013-01-01 17:52:47 +00:00
|
|
|
, remoteAnnexCostCommand :: Maybe String
|
|
|
|
, remoteAnnexIgnore :: Bool
|
|
|
|
, remoteAnnexSync :: Bool
|
2014-01-02 17:12:32 +00:00
|
|
|
, remoteAnnexReadOnly :: Bool
|
2013-01-01 17:52:47 +00:00
|
|
|
, remoteAnnexTrustLevel :: Maybe String
|
|
|
|
, remoteAnnexStartCommand :: Maybe String
|
|
|
|
, remoteAnnexStopCommand :: Maybe String
|
2014-01-13 18:41:10 +00:00
|
|
|
, remoteAnnexAvailability :: Maybe Availability
|
2014-01-26 17:03:25 +00:00
|
|
|
, remoteAnnexBare :: Maybe Bool
|
2013-01-01 17:52:47 +00:00
|
|
|
|
2013-04-04 19:46:33 +00:00
|
|
|
{- These settings are specific to particular types of remotes
|
|
|
|
- including special remotes. -}
|
2014-05-16 11:34:43 +00:00
|
|
|
, remoteAnnexShell :: Maybe String
|
2013-01-01 17:52:47 +00:00
|
|
|
, remoteAnnexSshOptions :: [String]
|
|
|
|
, remoteAnnexRsyncOptions :: [String]
|
2014-02-02 20:06:34 +00:00
|
|
|
, remoteAnnexRsyncUploadOptions :: [String]
|
|
|
|
, remoteAnnexRsyncDownloadOptions :: [String]
|
2013-04-13 22:10:49 +00:00
|
|
|
, remoteAnnexRsyncTransport :: [String]
|
2013-03-11 01:33:13 +00:00
|
|
|
, remoteAnnexGnupgOptions :: [String]
|
2013-01-01 17:52:47 +00:00
|
|
|
, remoteAnnexRsyncUrl :: Maybe String
|
|
|
|
, remoteAnnexBupRepo :: Maybe String
|
2014-01-08 20:14:37 +00:00
|
|
|
, remoteAnnexTahoe :: Maybe FilePath
|
2013-01-01 17:52:47 +00:00
|
|
|
, remoteAnnexBupSplitOptions :: [String]
|
|
|
|
, remoteAnnexDirectory :: Maybe FilePath
|
2013-09-24 21:25:47 +00:00
|
|
|
, remoteAnnexGCrypt :: Maybe String
|
2014-05-15 18:44:00 +00:00
|
|
|
, remoteAnnexDdarRepo :: Maybe String
|
2013-01-01 17:52:47 +00:00
|
|
|
, remoteAnnexHookType :: Maybe String
|
external special remotes mostly implemented (untested)
This has not been tested at all. It compiles!
The only known missing things are support for encryption, and for get/set
of special remote configuration, and of key state. (The latter needs
separate work to add a new per-key log file to store that state.)
Only thing I don't much like is that initremote needs to be passed both
type=external and externaltype=foo. It would be better to have just
type=foo
Most of this is quite straightforward code, that largely wrote itself given
the types. The only tricky parts were:
* Need to lock the remote when using it to eg make a request, because
in theory git-annex could have multiple threads that each try to use
a remote at the same time. I don't think that git-annex ever does
that currently, but better safe than sorry.
* Rather than starting up every external special remote program when
git-annex starts, they are started only on demand, when first used.
This will avoid slowdown, especially when running fast git-annex query
commands. Once started, they keep running until git-annex stops, currently,
which may not be ideal, but it's hard to know a better time to stop them.
* Bit of a chicken and egg problem with caching the cost of the remote,
because setting annex-cost in the git config needs the remote to already
be set up. Managed to finesse that.
This commit was sponsored by Lukas Anzinger.
2013-12-26 22:23:13 +00:00
|
|
|
, remoteAnnexExternalType :: Maybe String
|
2013-04-04 19:46:33 +00:00
|
|
|
{- A regular git remote's git repository config. -}
|
|
|
|
, remoteGitConfig :: Maybe GitConfig
|
2013-01-01 17:52:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
extractRemoteGitConfig :: Git.Repo -> String -> RemoteGitConfig
|
|
|
|
extractRemoteGitConfig r remotename = RemoteGitConfig
|
|
|
|
{ remoteAnnexCost = getmayberead "cost"
|
|
|
|
, remoteAnnexCostCommand = notempty $ getmaybe "cost-command"
|
|
|
|
, remoteAnnexIgnore = getbool "ignore" False
|
|
|
|
, remoteAnnexSync = getbool "sync" True
|
2014-01-02 17:12:32 +00:00
|
|
|
, remoteAnnexReadOnly = getbool "readonly" False
|
2013-01-01 17:52:47 +00:00
|
|
|
, remoteAnnexTrustLevel = notempty $ getmaybe "trustlevel"
|
|
|
|
, remoteAnnexStartCommand = notempty $ getmaybe "start-command"
|
|
|
|
, remoteAnnexStopCommand = notempty $ getmaybe "stop-command"
|
2014-01-13 18:41:10 +00:00
|
|
|
, remoteAnnexAvailability = getmayberead "availability"
|
2014-01-26 17:03:25 +00:00
|
|
|
, remoteAnnexBare = getmaybebool "bare"
|
2013-01-01 17:52:47 +00:00
|
|
|
|
2014-05-16 11:34:43 +00:00
|
|
|
, remoteAnnexShell = getmaybe "shell"
|
2013-01-01 17:52:47 +00:00
|
|
|
, remoteAnnexSshOptions = getoptions "ssh-options"
|
|
|
|
, remoteAnnexRsyncOptions = getoptions "rsync-options"
|
2014-02-02 20:06:34 +00:00
|
|
|
, remoteAnnexRsyncDownloadOptions = getoptions "rsync-download-options"
|
|
|
|
, remoteAnnexRsyncUploadOptions = getoptions "rsync-upload-options"
|
2013-04-13 22:10:49 +00:00
|
|
|
, remoteAnnexRsyncTransport = getoptions "rsync-transport"
|
2013-03-11 01:33:13 +00:00
|
|
|
, remoteAnnexGnupgOptions = getoptions "gnupg-options"
|
2013-01-01 17:52:47 +00:00
|
|
|
, remoteAnnexRsyncUrl = notempty $ getmaybe "rsyncurl"
|
|
|
|
, remoteAnnexBupRepo = getmaybe "buprepo"
|
2014-01-08 20:14:37 +00:00
|
|
|
, remoteAnnexTahoe = getmaybe "tahoe"
|
2013-01-01 17:52:47 +00:00
|
|
|
, remoteAnnexBupSplitOptions = getoptions "bup-split-options"
|
|
|
|
, remoteAnnexDirectory = notempty $ getmaybe "directory"
|
2013-09-24 21:25:47 +00:00
|
|
|
, remoteAnnexGCrypt = notempty $ getmaybe "gcrypt"
|
2014-05-15 18:44:00 +00:00
|
|
|
, remoteAnnexDdarRepo = getmaybe "ddarrepo"
|
2013-01-01 17:52:47 +00:00
|
|
|
, remoteAnnexHookType = notempty $ getmaybe "hooktype"
|
external special remotes mostly implemented (untested)
This has not been tested at all. It compiles!
The only known missing things are support for encryption, and for get/set
of special remote configuration, and of key state. (The latter needs
separate work to add a new per-key log file to store that state.)
Only thing I don't much like is that initremote needs to be passed both
type=external and externaltype=foo. It would be better to have just
type=foo
Most of this is quite straightforward code, that largely wrote itself given
the types. The only tricky parts were:
* Need to lock the remote when using it to eg make a request, because
in theory git-annex could have multiple threads that each try to use
a remote at the same time. I don't think that git-annex ever does
that currently, but better safe than sorry.
* Rather than starting up every external special remote program when
git-annex starts, they are started only on demand, when first used.
This will avoid slowdown, especially when running fast git-annex query
commands. Once started, they keep running until git-annex stops, currently,
which may not be ideal, but it's hard to know a better time to stop them.
* Bit of a chicken and egg problem with caching the cost of the remote,
because setting annex-cost in the git config needs the remote to already
be set up. Managed to finesse that.
This commit was sponsored by Lukas Anzinger.
2013-12-26 22:23:13 +00:00
|
|
|
, remoteAnnexExternalType = notempty $ getmaybe "externaltype"
|
2013-04-04 19:46:33 +00:00
|
|
|
, remoteGitConfig = Nothing
|
2013-01-01 17:52:47 +00:00
|
|
|
}
|
|
|
|
where
|
|
|
|
getbool k def = fromMaybe def $ getmaybebool k
|
|
|
|
getmaybebool k = Git.Config.isTrue =<< getmaybe k
|
|
|
|
getmayberead k = readish =<< getmaybe k
|
2013-04-03 07:52:41 +00:00
|
|
|
getmaybe k = mplus (Git.Config.getMaybe (key k) r)
|
|
|
|
(Git.Config.getMaybe (remotekey k) r)
|
2013-01-01 17:52:47 +00:00
|
|
|
getoptions k = fromMaybe [] $ words <$> getmaybe k
|
|
|
|
|
|
|
|
key k = "annex." ++ k
|
|
|
|
remotekey k = "remote." ++ remotename ++ ".annex-" ++ k
|
|
|
|
|
|
|
|
notempty :: Maybe String -> Maybe String
|
|
|
|
notempty Nothing = Nothing
|
|
|
|
notempty (Just "") = Nothing
|
|
|
|
notempty (Just s) = Just s
|
|
|
|
|