directory special remote: Made more efficient and robust.
Files are now written to a tmp directory in the remote, and once all chunks are written, etc, it's moved into the final place atomically. For now, checkpresent still checks every single chunk of a file, because the old method could leave partially transferred files with some chunks present and others not.
This commit is contained in:
parent
d3dfeeb3d9
commit
5f977cc725
5 changed files with 47 additions and 42 deletions
|
@ -58,33 +58,28 @@ chunkStream = map (\n -> ".chunk" ++ show n) [1 :: Integer ..]
|
|||
{- Given the base destination to use to store a value,
|
||||
- generates a stream of temporary destinations (just one when not chunking)
|
||||
- and passes it to an action, which should chunk and store the data,
|
||||
- and return the destinations it stored to, or [] on error.
|
||||
-
|
||||
- Then calles the finalizer to rename the temporary destinations into
|
||||
- their final places (and do any other cleanup), and writes the chunk count
|
||||
- (if chunking)
|
||||
- and return the destinations it stored to, or [] on error. Then
|
||||
- calls the storer to write the chunk count (if chunking). Finally, the
|
||||
- fianlizer is called to rename the tmp into the dest
|
||||
- (and do any other cleanup).
|
||||
-}
|
||||
storeChunks :: FilePath -> ChunkSize -> ([FilePath] -> IO [FilePath]) -> (FilePath -> String -> IO ()) -> (FilePath -> FilePath -> IO ()) -> IO Bool
|
||||
storeChunks basedest chunksize storer recorder finalizer =
|
||||
storeChunks :: Key -> FilePath -> FilePath -> ChunkSize -> ([FilePath] -> IO [FilePath]) -> (FilePath -> String -> IO ()) -> (FilePath -> FilePath -> IO ()) -> IO Bool
|
||||
storeChunks key tmp dest chunksize storer recorder finalizer =
|
||||
either (const $ return False) return
|
||||
=<< (E.try go :: IO (Either E.SomeException Bool))
|
||||
where
|
||||
go = do
|
||||
stored <- storer tmpdests
|
||||
forM_ stored $ \d -> do
|
||||
let dest = detmpprefix d
|
||||
finalizer d dest
|
||||
when (chunksize /= Nothing) $ do
|
||||
let chunkcount = basedest ++ chunkCount
|
||||
let chunkcount = basef ++ chunkCount
|
||||
recorder chunkcount (show $ length stored)
|
||||
finalizer tmp dest
|
||||
return (not $ null stored)
|
||||
|
||||
tmpprefix = ".tmp"
|
||||
detmpprefix f = take (length f - tmpprefixlen) f
|
||||
tmpprefixlen = length tmpprefix
|
||||
basef = tmp ++ keyFile key
|
||||
tmpdests
|
||||
| chunksize == Nothing = [basedest ++ tmpprefix]
|
||||
| otherwise = map (++ tmpprefix) $ map (basedest ++) chunkStream
|
||||
| chunksize == Nothing = [basef]
|
||||
| otherwise = map (basef ++ ) chunkStream
|
||||
|
||||
{- Given a list of destinations to use, chunks the data according to the
|
||||
- ChunkSize, and runs the storer action to store each chunk. Returns
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue