run download checksum verification in separate job pool

get, move, copy, sync: When -J or annex.jobs has enabled concurrency,
checksum verification uses a separate job pool than is used for
downloads, to keep bandwidth saturated.

Not yet done for upload checksum verification, but that only affects
remotes on local disks.
This commit is contained in:
Joey Hess 2019-06-17 14:58:02 -04:00
parent 5a9842d7ed
commit 04cc470201
No known key found for this signature in database
GPG key ID: DB12DB0FF05F8F38
8 changed files with 43 additions and 35 deletions

View file

@ -7,6 +7,9 @@ module Types.Concurrency where
import Utility.PartialPrelude
-- Note that Concurrent 1 is not the same as NonConcurrent;
-- the former specifies 1 job of each particular kind, but there can be
-- more than one kind of job running concurrently.
data Concurrency = NonConcurrent | Concurrent Int | ConcurrentPerCpu
parseConcurrency :: String -> Maybe Concurrency

View file

@ -14,6 +14,7 @@ import Control.Concurrent.Async
data WorkerPool t
= UnallocatedWorkerPool
| WorkerPool [Worker t]
deriving (Show)
-- | A worker can either be idle or running an Async action.
-- And it is used for some stage.
@ -21,9 +22,13 @@ data Worker t
= IdleWorker t WorkerStage
| ActiveWorker (Async t) WorkerStage
instance Show (Worker t) where
show (IdleWorker _ s) = "IdleWorker " ++ show s
show (ActiveWorker _ s) = "ActiveWorker " ++ show s
-- | These correspond to CommandPerform and CommandCleanup.
data WorkerStage = PerformStage | CleanupStage
deriving (Eq)
deriving (Show, Eq)
workerStage :: Worker t -> WorkerStage
workerStage (IdleWorker _ s) = s