directory special remote: Made more efficient and robust.

Files are now written to a tmp directory in the remote, and once all
chunks are written, etc, it's moved into the final place atomically.

For now, checkpresent still checks every single chunk of a file, because
the old method could leave partially transferred files with some chunks
present and others not.
This commit is contained in:
Joey Hess 2012-11-19 13:18:23 -04:00
parent d3dfeeb3d9
commit 5f977cc725
5 changed files with 47 additions and 42 deletions

View file

@ -48,7 +48,7 @@ gen r u c = do
storeKey = store dir chunksize,
retrieveKeyFile = retrieve dir chunksize,
retrieveKeyFileCheap = retrieveCheap dir chunksize,
removeKey = remove dir chunksize,
removeKey = remove dir,
hasKey = checkPresent dir chunksize,
hasKeyCheap = True,
whereisKey = Nothing,
@ -73,10 +73,19 @@ directorySetup u c = do
gitConfigSpecialRemote u c' "directory" dir
return $ M.delete "directory" c'
{- Locations to try to access a given Key in the Directory. -}
{- Locations to try to access a given Key in the Directory.
- We try more than since we used to write to different hash directories. -}
locations :: FilePath -> Key -> [FilePath]
locations d k = map (d </>) (keyPaths k)
{- Directory where the file(s) for a key are stored. -}
storeDir :: FilePath -> Key -> FilePath
storeDir d k = addTrailingPathSeparator $ d </> hashDirLower k </> keyFile k
{- Where we store temporary data for a key as it's being uploaded. -}
tmpDir :: FilePath -> Key -> FilePath
tmpDir d k = addTrailingPathSeparator $ d </> "tmp" </> keyFile k
withCheckedFiles :: (FilePath -> IO Bool) -> ChunkSize -> FilePath -> Key -> ([FilePath] -> IO Bool) -> IO Bool
withCheckedFiles _ _ [] _ _ = return False
withCheckedFiles check Nothing d k a = go $ locations d k
@ -159,18 +168,22 @@ storeSplit' meterupdate chunksize (d:dests) bs c = do
storeHelper :: FilePath -> ChunkSize -> Key -> ([FilePath] -> IO [FilePath]) -> Annex Bool
storeHelper d chunksize key storer = check <&&> go
where
basedest = Prelude.head $ locations d key
dir = parentDir basedest
tmpdir = tmpDir d key
destdir = storeDir d key
{- The size is not exactly known when encrypting the key;
- this assumes that at least the size of the key is
- needed as free space. -}
check = checkDiskSpace (Just dir) key 0
go = liftIO $ catchBoolIO $ do
createDirectoryIfMissing True dir
allowWrite dir
preventWrite dir `after` storeChunks basedest chunksize storer recorder finalizer
finalizer f dest = do
renameFile f dest
check = do
liftIO $ createDirectoryIfMissing True tmpdir
checkDiskSpace (Just tmpdir) key 0
go = liftIO $ catchBoolIO $
storeChunks key tmpdir destdir chunksize storer recorder finalizer
finalizer tmp dest = do
void $ tryIO $ allowWrite dest -- may already exist
void $ tryIO $ removeDirectoryRecursive dest -- or not exist
createDirectoryIfMissing True (parentDir dest)
renameDirectory tmp dest
mapM_ preventWrite =<< dirContents dest
preventWrite dest
recorder f s = do
void $ tryIO $ allowWrite f
@ -201,16 +214,13 @@ retrieveCheap d _ k f = liftIO $ withStoredFiles Nothing d k go
go [file] = catchBoolIO $ createSymbolicLink file f >> return True
go _files = return False
remove :: FilePath -> ChunkSize -> Key -> Annex Bool
remove d chunksize k = liftIO $ withStoredFiles chunksize d k go
remove :: FilePath -> Key -> Annex Bool
remove d k = liftIO $ catchBoolIO $ do
allowWrite dir
removeDirectoryRecursive dir
return True
where
go = all id <$$> mapM removefile
removefile file = catchBoolIO $ do
let dir = parentDir file
allowWrite dir
removeFile file
_ <- tryIO $ removeDirectory dir
return True
dir = storeDir d k
checkPresent :: FilePath -> ChunkSize -> Key -> Annex (Either String Bool)
checkPresent d chunksize k = liftIO $ catchMsgIO $ withStoredFiles chunksize d k $