2020-12-04 17:50:03 +00:00
|
|
|
{- serialized output
|
|
|
|
-
|
|
|
|
- Copyright 2020 Joey Hess <id@joeyh.name>
|
|
|
|
-
|
|
|
|
- Licensed under the GNU AGPL version 3 or higher.
|
|
|
|
-}
|
|
|
|
|
|
|
|
{-# LANGUAGE RankNTypes #-}
|
|
|
|
|
2020-12-04 18:54:09 +00:00
|
|
|
module Messages.Serialized (
|
|
|
|
relaySerializedOutput,
|
|
|
|
outputSerialized,
|
|
|
|
waitOutputSerializedResponse,
|
|
|
|
) where
|
2020-12-04 17:50:03 +00:00
|
|
|
|
|
|
|
import Common
|
|
|
|
import Annex
|
|
|
|
import Types.Messages
|
|
|
|
import Messages
|
|
|
|
import Messages.Internal
|
|
|
|
import Messages.Progress
|
|
|
|
import qualified Messages.JSON as JSON
|
2020-12-11 16:39:00 +00:00
|
|
|
import Utility.Metered (BytesProcessed, setMeterTotalSize)
|
2023-04-12 21:18:29 +00:00
|
|
|
import Git.Quote
|
2020-12-04 17:50:03 +00:00
|
|
|
|
|
|
|
import Control.Monad.IO.Class (MonadIO)
|
|
|
|
|
2020-12-04 18:54:09 +00:00
|
|
|
-- | Relay serialized output from a child process to the console.
|
2020-12-04 17:50:03 +00:00
|
|
|
relaySerializedOutput
|
|
|
|
:: (Monad m, MonadIO m, MonadMask m)
|
|
|
|
=> m (Either SerializedOutput r)
|
|
|
|
-- ^ Get next serialized output, or final value to return.
|
2020-12-04 18:54:09 +00:00
|
|
|
-> (SerializedOutputResponse -> m ())
|
2020-12-08 19:22:18 +00:00
|
|
|
-- ^ Send response to child process.
|
|
|
|
-> (Maybe BytesProcessed -> m ())
|
avoid using temp file size when deciding whether to retry failed transfer
When stall detection is enabled, and a transfer is in progress,
it would display a doubled message:
(transfer already in progress, or unable to take transfer lock) (transfer already in progress, or unable to take transfer lock)
That happened because the forward retry decider had a start size of 0,
and an end size of whatever amount of the object the other process had
downloaded. So it incorrectly thought that the transferrer process had
made progress, when it had in fact immediately given up with that
message.
Instead, use the reported value from the progress meter. If a remote
does not report progress, this will mean it doesn't forward retry, in a
situation where it used to. But most remotes do report progress, and any
remote that does not can be fixed to, by using watchFileSize when
downloading. Also, some remotes might preallocate the temp file (eg
bittorrent), so relying on statting its size at this level to get
progress is dubious.
The same change was made to Annex/Transfer.hs, although only
Annex/TransferrerPool.hs needed to be changed to avoid the duplicate
message.
(An alternate fix would have been to start the retry decider with the
size of the object file before downloading begins, rather than 0.)
Sponsored-by: Brett Eisenberg on Patreon
2021-06-25 15:53:28 +00:00
|
|
|
-- ^ When a progress meter is running, it is updated with
|
2020-12-08 19:22:18 +00:00
|
|
|
-- progress meter values sent by the process.
|
|
|
|
-- When a progress meter is stopped, Nothing is sent.
|
2020-12-04 17:50:03 +00:00
|
|
|
-> (forall a. Annex a -> m a)
|
|
|
|
-- ^ Run an annex action in the monad. Will not be used with
|
|
|
|
-- actions that block for a long time.
|
|
|
|
-> m r
|
2020-12-08 19:22:18 +00:00
|
|
|
relaySerializedOutput getso sendsor meterreport runannex = go Nothing
|
2020-12-04 17:50:03 +00:00
|
|
|
where
|
|
|
|
go st = loop st >>= \case
|
|
|
|
Right r -> return r
|
|
|
|
Left st' -> go st'
|
|
|
|
|
|
|
|
loop st = getso >>= \case
|
|
|
|
Right r -> return (Right r)
|
|
|
|
Left (OutputMessage msg) -> do
|
|
|
|
runannex $ outputMessage'
|
|
|
|
(\_ _ -> return False)
|
|
|
|
id
|
2023-04-10 21:03:41 +00:00
|
|
|
id
|
|
|
|
(UnquotedByteString msg)
|
2020-12-04 17:50:03 +00:00
|
|
|
loop st
|
|
|
|
Left (OutputError msg) -> do
|
filter out control characters in warning messages
Converted warning and similar to use StringContainingQuotedPath. Most
warnings are static strings, some do refer to filepaths that need to be
quoted, and others don't need quoting.
Note that, since quote filters out control characters of even
UnquotedString, this makes all warnings safe, even when an attacker
sneaks in a control character in some other way.
When json is being output, no quoting is done, since json gets its own
quoting.
This does, as a side effect, make warning messages in json output not
be indented. The indentation is only needed to offset warning messages
underneath the display of the file they apply to, so that's ok.
Sponsored-by: Brett Eisenberg on Patreon
2023-04-10 18:47:32 +00:00
|
|
|
runannex $ outputError id $ UnquotedString msg
|
2020-12-04 17:50:03 +00:00
|
|
|
loop st
|
|
|
|
Left (JSONObject b) -> do
|
|
|
|
runannex $ withMessageState $ \s -> case outputType s of
|
|
|
|
JSONOutput _ -> liftIO $ flushed $ JSON.emit' b
|
2020-12-04 18:54:09 +00:00
|
|
|
SerializedOutput h _ -> liftIO $
|
2020-12-04 17:50:03 +00:00
|
|
|
outputSerialized h $ JSONObject b
|
|
|
|
_ -> q
|
|
|
|
loop st
|
2020-12-11 16:52:22 +00:00
|
|
|
Left BeginProgressMeter -> do
|
2020-12-04 17:50:03 +00:00
|
|
|
ost <- runannex (Annex.getState Annex.output)
|
2021-06-08 16:48:30 +00:00
|
|
|
let setclear = const noop
|
2020-12-04 17:50:03 +00:00
|
|
|
-- Display a progress meter while running, until
|
|
|
|
-- the meter ends or a final value is returned.
|
bwlimit
Added annex.bwlimit and remote.name.annex-bwlimit config that works for git
remotes and many but not all special remotes.
This nearly works, at least for a git remote on the same disk. With it set
to 100kb/1s, the meter displays an actual bandwidth of 128 kb/s, with
occasional spikes to 160 kb/s. So it needs to delay just a bit longer...
I'm unsure why.
However, at the beginning a lot of data flows before it determines the
right bandwidth limit. A granularity of less than 1s would probably improve
that.
And, I don't know yet if it makes sense to have it be 100ks/1s rather than
100kb/s. Is there a situation where the user would want a larger
granularity? Does granulatity need to be configurable at all? I only used that
format for the config really in order to reuse an existing parser.
This can't support for external special remotes, or for ones that
themselves shell out to an external command. (Well, it could, but it
would involve pausing and resuming the child process tree, which seems
very hard to implement and very strange besides.) There could also be some
built-in special remotes that it still doesn't work for, due to them not
having a progress meter whose displays blocks the bandwidth using thread.
But I don't think there are actually any that run a separate thread for
downloads than the thread that displays the progress meter.
Sponsored-by: Graham Spencer on Patreon
2021-09-21 20:58:02 +00:00
|
|
|
metered' ost setclear Nothing Nothing Nothing (runannex showOutput)
|
2020-12-11 16:39:00 +00:00
|
|
|
(\meter meterupdate -> loop (Just (meter, meterupdate)))
|
2020-12-04 17:50:03 +00:00
|
|
|
>>= \case
|
|
|
|
Right r -> return (Right r)
|
|
|
|
-- Continue processing serialized
|
|
|
|
-- output after the progress meter
|
|
|
|
-- is done.
|
|
|
|
Left _st' -> loop Nothing
|
2020-12-08 19:22:18 +00:00
|
|
|
Left EndProgressMeter -> do
|
|
|
|
meterreport Nothing
|
|
|
|
return (Left st)
|
2020-12-04 17:50:03 +00:00
|
|
|
Left (UpdateProgressMeter n) -> do
|
|
|
|
case st of
|
2020-12-11 16:39:00 +00:00
|
|
|
Just (_, meterupdate) -> do
|
2020-12-08 19:22:18 +00:00
|
|
|
meterreport (Just n)
|
|
|
|
liftIO $ meterupdate n
|
2020-12-04 17:50:03 +00:00
|
|
|
Nothing -> noop
|
|
|
|
loop st
|
2020-12-11 16:39:00 +00:00
|
|
|
Left (UpdateProgressMeterTotalSize sz) -> do
|
|
|
|
case st of
|
|
|
|
Just (meter, _) -> liftIO $
|
|
|
|
setMeterTotalSize meter sz
|
|
|
|
Nothing -> noop
|
|
|
|
loop st
|
|
|
|
Left BeginPrompt -> do
|
2020-12-04 18:54:09 +00:00
|
|
|
prompter <- runannex mkPrompter
|
|
|
|
v <- prompter $ do
|
|
|
|
sendsor ReadyPrompt
|
|
|
|
-- Continue processing serialized output
|
|
|
|
-- until EndPrompt or a final value is
|
|
|
|
-- returned. (EndPrompt is all that
|
|
|
|
-- ought to be sent while in a prompt
|
|
|
|
-- really, but if something else did get
|
|
|
|
-- sent, display it just in case.)
|
|
|
|
loop st
|
|
|
|
case v of
|
|
|
|
Right r -> return (Right r)
|
|
|
|
Left st' -> loop st'
|
|
|
|
Left EndPrompt -> return (Left st)
|