add --size-limit option
When this option is not used, there should be effectively no added
overhead, thanks to the optimisation in
b3cd0cc6ba.
When an action fails on a file, the size of the file still counts toward
the size limit. This was necessary to support concurrency, but also
generally seems like the right choice.
Most commands that operate on annexed files support the option.
export and import do not, and I don't know if it would make sense for
export to.. Why would you want an incomplete export? sync doesn't, and
while it would be easy to make it support it for transferring files,
it's not clear if dropping files should also take the size limit into
account. Commands like add that don't operate on annexed files don't
support the option either.
Exiting 101 not yet implemented.
Sponsored-by: Denis Dzyubenko on Patreon
This commit is contained in:
Joey Hess
parent
b3cd0cc6ba
commit
771a122c9e
6 changed files with 118 additions and 19 deletions
|
|
@ -1,6 +1,6 @@
|
|||
{- git-annex command-line actions and concurrency
|
||||
-
|
||||
- Copyright 2010-2020 Joey Hess <id@joeyh.name>
|
||||
- Copyright 2010-2021 Joey Hess <id@joeyh.name>
|
||||
-
|
||||
- Licensed under the GNU AGPL version 3 or higher.
|
||||
-}
|
||||
|
|
@ -15,9 +15,11 @@ import Annex.Concurrent
|
|||
import Annex.WorkerPool
|
||||
import Types.Command
|
||||
import Types.Concurrency
|
||||
import Annex.Content
|
||||
import Messages.Concurrent
|
||||
import Types.Messages
|
||||
import Types.WorkerPool
|
||||
import Types.ActionItem
|
||||
import Remote.List
|
||||
|
||||
import Control.Concurrent
|
||||
|
|
@ -58,21 +60,29 @@ commandAction :: CommandStart -> Annex ()
|
|||
commandAction start = do
|
||||
st <- Annex.getState id
|
||||
case getConcurrency' (Annex.concurrency st) of
|
||||
NonConcurrent -> runnonconcurrent
|
||||
NonConcurrent -> runnonconcurrent (Annex.sizelimit st)
|
||||
Concurrent n
|
||||
| n > 1 -> runconcurrent (Annex.workers st)
|
||||
| otherwise -> runnonconcurrent
|
||||
ConcurrentPerCpu -> runconcurrent (Annex.workers st)
|
||||
| n > 1 -> runconcurrent (Annex.sizelimit st) (Annex.workers st)
|
||||
| otherwise -> runnonconcurrent (Annex.sizelimit st)
|
||||
ConcurrentPerCpu -> runconcurrent (Annex.sizelimit st) (Annex.workers st)
|
||||
where
|
||||
runnonconcurrent = void $ includeCommandAction start
|
||||
runconcurrent Nothing = runnonconcurrent
|
||||
runconcurrent (Just tv) =
|
||||
liftIO (atomically (waitStartWorkerSlot tv)) >>=
|
||||
maybe runnonconcurrent (runconcurrent' tv)
|
||||
runconcurrent' tv (workerstrd, workerstage) = do
|
||||
runnonconcurrent sizelimit = start >>= \case
|
||||
Nothing -> noop
|
||||
Just (startmsg, perform) ->
|
||||
checkSizeLimit sizelimit startmsg $ do
|
||||
showStartMessage startmsg
|
||||
void $ accountCommandAction startmsg $
|
||||
performCommandAction' startmsg perform
|
||||
|
||||
runconcurrent sizelimit Nothing = runnonconcurrent sizelimit
|
||||
runconcurrent sizelimit (Just tv) =
|
||||
liftIO (atomically (waitStartWorkerSlot tv)) >>= maybe
|
||||
(runnonconcurrent sizelimit)
|
||||
(runconcurrent' sizelimit tv)
|
||||
runconcurrent' sizelimit tv (workerstrd, workerstage) = do
|
||||
aid <- liftIO $ async $ snd
|
||||
<$> Annex.run workerstrd
|
||||
(concurrentjob (fst workerstrd))
|
||||
(concurrentjob sizelimit (fst workerstrd))
|
||||
liftIO $ atomically $ do
|
||||
pool <- takeTMVar tv
|
||||
let !pool' = addWorkerPool (ActiveWorker aid workerstage) pool
|
||||
|
|
@ -88,10 +98,11 @@ commandAction start = do
|
|||
let !pool' = deactivateWorker pool aid workerstrd'
|
||||
putTMVar tv pool'
|
||||
|
||||
concurrentjob workerst = start >>= \case
|
||||
concurrentjob sizelimit workerst = start >>= \case
|
||||
Nothing -> noop
|
||||
Just (startmsg, perform) ->
|
||||
concurrentjob' workerst startmsg perform
|
||||
checkSizeLimit sizelimit startmsg $
|
||||
concurrentjob' workerst startmsg perform
|
||||
|
||||
concurrentjob' workerst startmsg perform = case mkActionItem startmsg of
|
||||
OnlyActionOn k _ -> ensureOnlyActionOn k $
|
||||
|
|
@ -126,7 +137,7 @@ commandAction start = do
|
|||
Nothing -> do
|
||||
showEndMessage startmsg False
|
||||
return False
|
||||
|
||||
|
||||
{- Waits for all worker threads to finish and merges their AnnexStates
|
||||
- back into the current Annex's state.
|
||||
-}
|
||||
|
|
@ -294,3 +305,26 @@ ensureOnlyActionOn k a = debugLocks $
|
|||
writeTVar tv $! M.insert k mytid m
|
||||
return $ liftIO $ atomically $
|
||||
modifyTVar tv $ M.delete k
|
||||
|
||||
checkSizeLimit :: Maybe (TVar Integer) -> StartMessage -> Annex () -> Annex ()
|
||||
checkSizeLimit Nothing _ a = a
|
||||
checkSizeLimit (Just sizelimitvar) startmsg a =
|
||||
case actionItemKey (mkActionItem startmsg) of
|
||||
Just k -> case fromKey keySize k of
|
||||
Just sz -> go sz
|
||||
Nothing -> do
|
||||
fsz <- catchMaybeIO $ withObjectLoc k $
|
||||
liftIO . getFileSize
|
||||
maybe noop go fsz
|
||||
Nothing -> a
|
||||
where
|
||||
go sz = do
|
||||
fits <- liftIO $ atomically $ do
|
||||
n <- readTVar sizelimitvar
|
||||
let !n' = n - sz
|
||||
if n' >= 0
|
||||
then do
|
||||
writeTVar sizelimitvar n'
|
||||
return True
|
||||
else return False
|
||||
when fits a
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ module CmdLine.GitAnnex.Options where
|
|||
import Control.Monad.Fail as Fail (MonadFail(..))
|
||||
import Options.Applicative
|
||||
import Data.Time.Clock.POSIX
|
||||
import Control.Concurrent.STM
|
||||
import qualified Data.Map as M
|
||||
|
||||
import Annex.Common
|
||||
|
|
@ -37,6 +38,7 @@ import CmdLine.GlobalSetter
|
|||
import qualified Backend
|
||||
import qualified Types.Backend as Backend
|
||||
import Utility.HumanTime
|
||||
import Utility.DataUnits
|
||||
import Annex.Concurrent
|
||||
|
||||
-- Global options that are accepted by all git-annex sub-commands,
|
||||
|
|
@ -233,11 +235,12 @@ annexedMatchingOptions = concat
|
|||
, fileMatchingOptions' Limit.LimitAnnexFiles
|
||||
, combiningOptions
|
||||
, timeLimitOption
|
||||
, sizeLimitOption
|
||||
]
|
||||
|
||||
-- Matching options that can operate on keys as well as files.
|
||||
keyMatchingOptions :: [GlobalOption]
|
||||
keyMatchingOptions = keyMatchingOptions' ++ combiningOptions ++ timeLimitOption
|
||||
keyMatchingOptions = keyMatchingOptions' ++ combiningOptions ++ timeLimitOption ++ sizeLimitOption
|
||||
|
||||
keyMatchingOptions' :: [GlobalOption]
|
||||
keyMatchingOptions' =
|
||||
|
|
@ -435,6 +438,19 @@ timeLimitOption =
|
|||
let cutoff = start + durationToPOSIXTime duration
|
||||
Annex.changeState $ \s -> s { Annex.timelimit = Just (duration, cutoff) }
|
||||
|
||||
sizeLimitOption :: [GlobalOption]
|
||||
sizeLimitOption =
|
||||
[ globalOption setsizelimit $ option (maybeReader (readSize dataUnits))
|
||||
( long "size-limit" <> metavar paramSize
|
||||
<> help "total size of annexed files to process"
|
||||
<> hidden
|
||||
)
|
||||
]
|
||||
where
|
||||
setsizelimit n = setAnnexState $ do
|
||||
v <- liftIO $ newTVarIO n
|
||||
Annex.changeState $ \s -> s { Annex.sizelimit = Just v }
|
||||
|
||||
data DaemonOptions = DaemonOptions
|
||||
{ foregroundDaemonOption :: Bool
|
||||
, stopDaemonOption :: Bool
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue