use total size from DATA

Noticed that getting a key whose size is not known resulted in a
progress display that didn't include the percent complete.

Fixed for P2P by making the size sent with DATA be used to update the
meter's total size.

In order for rateLimitMeterUpdate to also learn the total size,
had to make it be passed the Meter, and some other reorg in
Utility.Metered was also done so that --json-progress can construct a
Meter to pass to rateLimitMeterUpdate.

When the fallback rsync is done, the progress display still doesn't
include the percent complete. Only way to fix that seems to be to let rsync
display its output again, but that would conflict with git-annex's
own progress meter, which is also being displayed.

This commit was sponsored by Henrik Riomar on Patreon.
This commit is contained in:
Joey Hess 2018-03-12 21:46:58 -04:00
parent b96b845ffd
commit e16b069331
No known key found for this signature in database
GPG key ID: DB12DB0FF05F8F38
7 changed files with 68 additions and 47 deletions

View file

@ -220,7 +220,7 @@ performExport r ea db ek af contentsha loc = do
let rollback = void $
performUnexport r ea db [ek] loc
sendAnnex k rollback $ \f ->
metered Nothing k (return $ Just f) $ \m -> do
metered Nothing k (return $ Just f) $ \_ m -> do
let m' = combineMeterUpdate pm m
storer f k loc m'
, do
@ -228,7 +228,7 @@ performExport r ea db ek af contentsha loc = do
return False
)
-- Sending a non-annexed file.
GitKey sha1k -> metered Nothing sha1k (return Nothing) $ \m ->
GitKey sha1k -> metered Nothing sha1k (return Nothing) $ \_ m ->
withTmpFile "export" $ \tmp h -> do
b <- catObject contentsha
liftIO $ L.hPut h b

View file

@ -24,46 +24,52 @@ import qualified System.Console.Concurrent as Console
#endif
{- Shows a progress meter while performing a transfer of a key.
- The action is passed a callback to use to update the meter.
- The action is passed the meter and a callback to use to update the meter.
-
- When the key's size is not known, the srcfile is statted to get the size.
- This allows uploads of keys without size to still have progress
- displayed.
--}
metered :: Maybe MeterUpdate -> Key -> Annex (Maybe FilePath) -> (MeterUpdate -> Annex a) -> Annex a
metered :: Maybe MeterUpdate -> Key -> Annex (Maybe FilePath) -> (Meter -> MeterUpdate -> Annex a) -> Annex a
metered othermeter key getsrcfile a = withMessageState $ \st ->
flip go st =<< getsz
where
go _ (MessageState { outputType = QuietOutput }) = nometer
go msize (MessageState { outputType = NormalOutput, concurrentOutputEnabled = False }) = do
showOutput
meter <- liftIO $ mkMeter msize bandwidthMeter $
displayMeterHandle stdout
m <- liftIO $ rateLimitMeterUpdate 0.1 msize $
meter <- liftIO $ mkMeter msize $
displayMeterHandle stdout bandwidthMeter
m <- liftIO $ rateLimitMeterUpdate 0.1 meter $
updateMeter meter
r <- a (combinemeter m)
r <- a meter (combinemeter m)
liftIO $ clearMeterHandle meter stdout
return r
go msize (MessageState { outputType = NormalOutput, concurrentOutputEnabled = True }) =
#if WITH_CONCURRENTOUTPUT
withProgressRegion $ \r -> do
meter <- liftIO $ mkMeter msize bandwidthMeter $ \_ s ->
Regions.setConsoleRegion r ('\n' : s)
m <- liftIO $ rateLimitMeterUpdate 0.1 msize $
meter <- liftIO $ mkMeter msize $ \_ msize' old new ->
let s = bandwidthMeter msize' old new
in Regions.setConsoleRegion r ('\n' : s)
m <- liftIO $ rateLimitMeterUpdate 0.1 meter $
updateMeter meter
a (combinemeter m)
a meter (combinemeter m)
#else
nometer
#endif
go msize (MessageState { outputType = JSONOutput jsonoptions })
| jsonProgress jsonoptions = do
buf <- withMessageState $ return . jsonBuffer
m <- liftIO $ rateLimitMeterUpdate 0.1 msize $
JSON.progress buf msize
a (combinemeter m)
meter <- liftIO $ mkMeter msize $ \_ msize' _old (new, _now) ->
JSON.progress buf msize' new
m <- liftIO $ rateLimitMeterUpdate 0.1 meter $
updateMeter meter
a meter (combinemeter m)
| otherwise = nometer
nometer = a $ combinemeter (const noop)
nometer = do
dummymeter <- liftIO $ mkMeter Nothing $
\_ _ _ _ -> return ()
a dummymeter (combinemeter (const noop))
combinemeter m = case othermeter of
Nothing -> m
@ -82,7 +88,7 @@ metered othermeter key getsrcfile a = withMessageState $ \st ->
meteredFile :: FilePath -> Maybe MeterUpdate -> Key -> Annex a -> Annex a
meteredFile file combinemeterupdate key a =
withMessageState $ \s -> if needOutputMeter s
then metered combinemeterupdate key (return Nothing) $ \p ->
then metered combinemeterupdate key (return Nothing) $ \_ p ->
watchFileSize file p a
else a
where

View file

@ -117,6 +117,9 @@ runLocal runst runner a = case a of
Left e -> return (Left (show e))
Right changedrefs -> runner (next changedrefs)
_ -> return $ Left "change notification not available"
UpdateMeterTotalSize m sz next -> do
liftIO $ setMeterTotalSize m sz
runner next
where
transfer mk k af ta = case runst of
-- Update transfer logs when serving.

View file

@ -257,7 +257,10 @@ data LocalF c
-- action. If unable to lock the content, or the content is not
-- present, runs the protocol action with False.
| WaitRefChange (ChangedRefs -> c)
-- ^ Waits for one or more git refs to change and returns them.
-- ^ Waits for one or more git refs to change and returns them.a
| UpdateMeterTotalSize Meter Integer c
-- ^ Updates the total size of a Meter, for cases where the size is
-- not known until the data is being received.
deriving (Functor)
type Local = Free LocalF
@ -323,8 +326,9 @@ remove key = do
net $ sendMessage (REMOVE key)
checkSuccess
get :: FilePath -> Key -> AssociatedFile -> MeterUpdate -> Proto Bool
get dest key af p = receiveContent p sizer storer (\offset -> GET offset af key)
get :: FilePath -> Key -> AssociatedFile -> Meter -> MeterUpdate -> Proto Bool
get dest key af m p =
receiveContent (Just m) p sizer storer (\offset -> GET offset af key)
where
sizer = fileSize dest
storer = storeContentTo dest
@ -433,7 +437,7 @@ serveAuthed servermode myuuid = void $ serverLoop handler
else do
let sizer = tmpContentSize key
let storer = storeContent key af
ok <- receiveContent nullMeterUpdate sizer storer PUT_FROM
ok <- receiveContent Nothing nullMeterUpdate sizer storer PUT_FROM
when ok $
local $ setPresent key myuuid
return ServerContinue
@ -477,15 +481,18 @@ sendContent key af offset@(Offset n) p = go =<< local (contentSize key)
net $ sendBytes len content p'
checkSuccess
receiveContent :: MeterUpdate -> Local Len -> (Offset -> Len -> Proto L.ByteString -> Local Bool) -> (Offset -> Message) -> Proto Bool
receiveContent p sizer storer mkmsg = do
receiveContent :: Maybe Meter -> MeterUpdate -> Local Len -> (Offset -> Len -> Proto L.ByteString -> Local Bool) -> (Offset -> Message) -> Proto Bool
receiveContent mm p sizer storer mkmsg = do
Len n <- local sizer
let p' = offsetMeterUpdate p (toBytesProcessed n)
let offset = Offset n
net $ sendMessage (mkmsg offset)
r <- net receiveMessage
case r of
Just (DATA len) -> do
Just (DATA len@(Len l)) -> do
local $ case mm of
Nothing -> return ()
Just m -> updateMeterTotalSize m (n+l)
ok <- local $ storer offset len
(net (receiveBytes len p'))
sendSuccess ok

View file

@ -33,14 +33,14 @@ type WithConn a c = (ClosableConnection c -> Annex (ClosableConnection c, a)) ->
store :: (MeterUpdate -> ProtoRunner Bool) -> Key -> AssociatedFile -> MeterUpdate -> Annex Bool
store runner k af p = do
let getsrcfile = fmap fst <$> prepSendAnnex k
metered (Just p) k getsrcfile $ \p' ->
metered (Just p) k getsrcfile $ \_ p' ->
fromMaybe False
<$> runner p' (P2P.put k af p')
retrieve :: (MeterUpdate -> ProtoRunner Bool) -> Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex (Bool, Verification)
retrieve runner k af dest p = unVerified $
metered (Just p) k (return Nothing) $ \p' -> fromMaybe False
<$> runner p (P2P.get dest k af p')
metered (Just p) k (return Nothing) $ \m p' -> fromMaybe False
<$> runner p' (P2P.get dest k af m p')
remove :: ProtoRunner Bool -> Key -> Annex Bool
remove runner k = fromMaybe False <$> runner (P2P.remove k)

View file

@ -228,7 +228,7 @@ specialRemote' cfg c preparestorer prepareretriever prepareremover preparecheckp
chunkconfig = chunkConfig cfg
displayprogress p k srcfile a
| displayProgress cfg = metered (Just p) k (return srcfile) a
| displayProgress cfg = metered (Just p) k (return srcfile) (const a)
| otherwise = a p
{- Sink callback for retrieveChunks. Stores the file content into the

View file

@ -1,6 +1,6 @@
{- Metered IO and actions
-
- Copyright 2012-2016 Joey Hess <id@joeyh.name>
- Copyright 2012-2018 Joey Hess <id@joeyh.name>
-
- License: BSD-2-clause
-}
@ -288,14 +288,14 @@ outputFilter cmd params environ outfilter errfilter = catchBoolIO $ do
-- | Limit a meter to only update once per unit of time.
--
-- It's nice to display the final update to 100%, even if it comes soon
-- after a previous update. To make that happen, a total size has to be
-- provided.
rateLimitMeterUpdate :: NominalDiffTime -> Maybe Integer -> MeterUpdate -> IO MeterUpdate
rateLimitMeterUpdate delta totalsize meterupdate = do
-- after a previous update. To make that happen, the Meter has to know
-- its total size.
rateLimitMeterUpdate :: NominalDiffTime -> Meter -> MeterUpdate -> IO MeterUpdate
rateLimitMeterUpdate delta (Meter totalsizev _ _ _) meterupdate = do
lastupdate <- newMVar (toEnum 0 :: POSIXTime)
return $ mu lastupdate
where
mu lastupdate n@(BytesProcessed i) = case totalsize of
mu lastupdate n@(BytesProcessed i) = tryReadMVar totalsizev >>= \case
Just t | i >= t -> meterupdate n
_ -> do
now <- getPOSIXTime
@ -306,35 +306,40 @@ rateLimitMeterUpdate delta totalsize meterupdate = do
meterupdate n
else putMVar lastupdate prev
data Meter = Meter (Maybe Integer) (MVar MeterState) (MVar String) RenderMeter DisplayMeter
data Meter = Meter (MVar Integer) (MVar MeterState) (MVar String) DisplayMeter
type MeterState = (BytesProcessed, POSIXTime)
type DisplayMeter = MVar String -> String -> IO ()
type DisplayMeter = MVar String -> Maybe Integer -> (BytesProcessed, POSIXTime) -> (BytesProcessed, POSIXTime) -> IO ()
type RenderMeter = Maybe Integer -> (BytesProcessed, POSIXTime) -> (BytesProcessed, POSIXTime) -> String
-- | Make a meter. Pass the total size, if it's known.
mkMeter :: Maybe Integer -> RenderMeter -> DisplayMeter -> IO Meter
mkMeter totalsize rendermeter displaymeter = Meter
<$> pure totalsize
mkMeter :: Maybe Integer -> DisplayMeter -> IO Meter
mkMeter totalsize displaymeter = Meter
<$> maybe newEmptyMVar newMVar totalsize
<*> ((\t -> newMVar (zeroBytesProcessed, t)) =<< getPOSIXTime)
<*> newMVar ""
<*> pure rendermeter
<*> pure displaymeter
setMeterTotalSize :: Meter -> Integer -> IO ()
setMeterTotalSize (Meter totalsizev _ _ _) totalsize = do
void $ tryTakeMVar totalsizev
putMVar totalsizev totalsize
-- | Updates the meter, displaying it if necessary.
updateMeter :: Meter -> BytesProcessed -> IO ()
updateMeter (Meter totalsize sv bv rendermeter displaymeter) new = do
updateMeter (Meter totalsizev sv bv displaymeter) new = do
now <- getPOSIXTime
(old, before) <- swapMVar sv (new, now)
when (old /= new) $
displaymeter bv $
rendermeter totalsize (old, before) (new, now)
when (old /= new) $ do
totalsize <- tryReadMVar totalsizev
displaymeter bv totalsize (old, before) (new, now)
-- | Display meter to a Handle.
displayMeterHandle :: Handle -> DisplayMeter
displayMeterHandle h v s = do
displayMeterHandle :: Handle -> RenderMeter -> DisplayMeter
displayMeterHandle h rendermeter v msize old new = do
let s = rendermeter msize old new
olds <- swapMVar v s
-- Avoid writing when the rendered meter has not changed.
when (olds /= s) $ do
@ -344,7 +349,7 @@ displayMeterHandle h v s = do
-- | Clear meter displayed by displayMeterHandle.
clearMeterHandle :: Meter -> Handle -> IO ()
clearMeterHandle (Meter _ _ v _ _) h = do
clearMeterHandle (Meter _ _ v _) h = do
olds <- readMVar v
hPutStr h $ '\r' : replicate (length olds) ' ' ++ "\r"
hFlush h