core implementation of new style chunking
Not yet used by any special remotes, but should not be too hard to add it to most of them. storeChunks is the hairy bit! It's loosely based on Remote.Directory.storeLegacyChunked. The object is read in using a lazy bytestring, which is streamed though, creating chunks as needed, without ever buffering more than 1 chunk in memory. Getting the progress meter update to work right was also fun, since progress meter values are absolute. Finessed by constructing an offset meter. This commit was sponsored by Richard Collins.
This commit is contained in:
parent
8f93982df6
commit
ab4cce4114
2 changed files with 119 additions and 1 deletions
|
@ -16,6 +16,7 @@ import qualified Data.ByteString as S
|
|||
import System.IO.Unsafe
|
||||
import Foreign.Storable (Storable(sizeOf))
|
||||
import System.Posix.Types
|
||||
import Data.Int
|
||||
|
||||
{- An action that can be run repeatedly, updating it on the bytes processed.
|
||||
-
|
||||
|
@ -31,6 +32,10 @@ class AsBytesProcessed a where
|
|||
toBytesProcessed :: a -> BytesProcessed
|
||||
fromBytesProcessed :: BytesProcessed -> a
|
||||
|
||||
instance AsBytesProcessed BytesProcessed where
|
||||
toBytesProcessed = id
|
||||
fromBytesProcessed = id
|
||||
|
||||
instance AsBytesProcessed Integer where
|
||||
toBytesProcessed i = BytesProcessed i
|
||||
fromBytesProcessed (BytesProcessed i) = i
|
||||
|
@ -39,6 +44,10 @@ instance AsBytesProcessed Int where
|
|||
toBytesProcessed i = BytesProcessed $ toInteger i
|
||||
fromBytesProcessed (BytesProcessed i) = fromInteger i
|
||||
|
||||
instance AsBytesProcessed Int64 where
|
||||
toBytesProcessed i = BytesProcessed $ toInteger i
|
||||
fromBytesProcessed (BytesProcessed i) = fromInteger i
|
||||
|
||||
instance AsBytesProcessed FileOffset where
|
||||
toBytesProcessed sz = BytesProcessed $ toInteger sz
|
||||
fromBytesProcessed (BytesProcessed sz) = fromInteger sz
|
||||
|
@ -77,6 +86,13 @@ meteredWriteFile :: MeterUpdate -> FilePath -> L.ByteString -> IO ()
|
|||
meteredWriteFile meterupdate f b = withBinaryFile f WriteMode $ \h ->
|
||||
meteredWrite meterupdate h b
|
||||
|
||||
{- Applies an offset to a MeterUpdate. This can be useful when
|
||||
- performing a sequence of actions, such as multiple meteredWriteFiles,
|
||||
- that all update a common meter progressively.
|
||||
-}
|
||||
offsetMeterUpdate :: MeterUpdate -> BytesProcessed -> MeterUpdate
|
||||
offsetMeterUpdate base offset = \n -> base (offset `addBytesProcessed` n)
|
||||
|
||||
{- This is like L.hGetContents, but after each chunk is read, a meter
|
||||
- is updated based on the size of the chunk.
|
||||
-
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue