From 6ba3dea566db9777d1e2237c4e1f14c6616d284c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 4 Oct 2018 12:47:27 -0400 Subject: [PATCH] annex.jobs Added annex.jobs setting, which is like using the -J option. Of course, -J overrides annex.jobs. This commit was sponsored by Trenton Cronholm on Patreon. --- CHANGELOG | 1 + CmdLine/Action.hs | 34 +++++++++++++------ CmdLine/GitAnnex/Options.hs | 7 +--- Types/GitConfig.hs | 3 ++ doc/git-annex.mdwn | 7 ++++ ..._1fa91968840bb25bd09c6dfe64f1f8cd._comment | 11 ++++++ 6 files changed, 46 insertions(+), 17 deletions(-) create mode 100644 doc/todo/config_option_to_use_all_processors_by_default/comment_1_1fa91968840bb25bd09c6dfe64f1f8cd._comment diff --git a/CHANGELOG b/CHANGELOG index 4ddaa1adab..07e910a919 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -3,6 +3,7 @@ git-annex (6.20180927) UNRELEASED; urgency=medium * sync: Warn when a remote's export is not updated to the current tree because export tracking is not configured. * Improve display when git config download from a http remote fails. + * Added annex.jobs setting, which is like using the -J option. -- Joey Hess Thu, 27 Sep 2018 15:27:20 -0400 diff --git a/CmdLine/Action.hs b/CmdLine/Action.hs index d4775a1a7d..2abf902053 100644 --- a/CmdLine/Action.hs +++ b/CmdLine/Action.hs @@ -170,20 +170,32 @@ callCommandActionQuiet = start {- Do concurrent output when that has been requested. -} allowConcurrentOutput :: Annex a -> Annex a #ifdef WITH_CONCURRENTOUTPUT -allowConcurrentOutput a = go =<< Annex.getState Annex.concurrency +allowConcurrentOutput a = do + fromcmdline <- Annex.getState Annex.concurrency + fromgitcfg <- annexJobs <$> Annex.getGitConfig + case (fromcmdline, fromgitcfg) of + (NonConcurrent, NonConcurrent) -> a + (Concurrent n, _) -> goconcurrent n + (NonConcurrent, Concurrent n) -> do + Annex.changeState $ + \c -> c { Annex.concurrency = fromgitcfg } + goconcurrent n where - go NonConcurrent = a - go (Concurrent _) = ifM (liftIO concurrentOutputSupported) - ( Regions.displayConsoleRegions $ - goconcurrent True - , goconcurrent False - ) - goconcurrent b = bracket_ (setup b) cleanup a - setup = setconcurrentenabled + goconcurrent n = do + c <- liftIO getNumCapabilities + when (n > c) $ + liftIO $ setNumCapabilities n + ifM (liftIO concurrentOutputSupported) + ( Regions.displayConsoleRegions $ + goconcurrent' True + , goconcurrent' False + ) + goconcurrent' b = bracket_ (setup b) cleanup a + setup = setconcurrentoutputenabled cleanup = do finishCommandActions - setconcurrentenabled False - setconcurrentenabled b = Annex.changeState $ \s -> + setconcurrentoutputenabled False + setconcurrentoutputenabled b = Annex.changeState $ \s -> s { Annex.output = (Annex.output s) { concurrentOutputEnabled = b } } #else allowConcurrentOutput = id diff --git a/CmdLine/GitAnnex/Options.hs b/CmdLine/GitAnnex/Options.hs index 791f499d13..c94350d719 100644 --- a/CmdLine/GitAnnex/Options.hs +++ b/CmdLine/GitAnnex/Options.hs @@ -13,7 +13,6 @@ import Options.Applicative #if ! MIN_VERSION_optparse_applicative(0,14,1) import Options.Applicative.Builder.Internal #endif -import Control.Concurrent import qualified Data.Map as M import Annex.Common @@ -370,11 +369,7 @@ jobsOption = ) ] where - set n = do - Annex.changeState $ \s -> s { Annex.concurrency = Concurrent n } - c <- liftIO getNumCapabilities - when (n > c) $ - liftIO $ setNumCapabilities n + set n = Annex.changeState $ \s -> s { Annex.concurrency = Concurrent n } timeLimitOption :: [GlobalOption] timeLimitOption = diff --git a/Types/GitConfig.hs b/Types/GitConfig.hs index d46c8f1d45..63e0d77f4e 100644 --- a/Types/GitConfig.hs +++ b/Types/GitConfig.hs @@ -26,6 +26,7 @@ import Config.Cost import Types.UUID import Types.Distribution import Types.Availability +import Types.Concurrency import Types.NumCopies import Types.Difference import Types.RefSpec @@ -99,6 +100,7 @@ data GitConfig = GitConfig , annexAllowedHttpAddresses :: String , annexAllowUnverifiedDownloads :: Bool , annexMaxExtensionLength :: Maybe Int + , annexJobs :: Concurrency , coreSymlinks :: Bool , coreSharedRepository :: SharedRepository , receiveDenyCurrentBranch :: DenyCurrentBranch @@ -173,6 +175,7 @@ extractGitConfig r = GitConfig , annexAllowUnverifiedDownloads = (== Just "ACKTHPPT") $ getmaybe (annex "security.allow-unverified-downloads") , annexMaxExtensionLength = getmayberead (annex "maxextensionlength") + , annexJobs = maybe NonConcurrent Concurrent $ getmayberead (annex "jobs") , coreSymlinks = getbool "core.symlinks" True , coreSharedRepository = getSharedRepository r , receiveDenyCurrentBranch = getDenyCurrentBranch r diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index b4c1958827..fa8a6f3aac 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -935,6 +935,13 @@ Here are all the supported configuration settings. This controls which refs `git-annex unused` considers to be used. See REFSPEC FORMAT in [[git-annex-unused]](1) for details. +* `annex.jobs + + Configure the number of concurrent jobs to run. Default is 1. + + Only git-annex commands that support the --jobs option will + use this. + * `annex.queuesize` git-annex builds a queue of git commands, in order to combine similar diff --git a/doc/todo/config_option_to_use_all_processors_by_default/comment_1_1fa91968840bb25bd09c6dfe64f1f8cd._comment b/doc/todo/config_option_to_use_all_processors_by_default/comment_1_1fa91968840bb25bd09c6dfe64f1f8cd._comment new file mode 100644 index 0000000000..cc79e00c62 --- /dev/null +++ b/doc/todo/config_option_to_use_all_processors_by_default/comment_1_1fa91968840bb25bd09c6dfe64f1f8cd._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2018-10-04T16:04:57Z" + content=""" +git-annex is rarely cpu-bound so I don't know how useful that would really +be in practice. + +A git config to set the concurrency level is certianly a good idea, +implemented as annex.jobs. +"""]]