git-annex/Remote/Directory.hs

605 lines
22 KiB
Haskell
Raw Normal View History

{- A "remote" that is just a filesystem directory.
2011-03-30 17:18:46 +00:00
-
- Copyright 2011-2022 Joey Hess <id@joeyh.name>
2011-03-30 17:18:46 +00:00
-
- Licensed under the GNU AGPL version 3 or higher.
2011-03-30 17:18:46 +00:00
-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE RankNTypes #-}
{-# LANGUAGE CPP #-}
module Remote.Directory (
remote,
finalizeStoreGeneric,
removeDirGeneric,
) where
2011-03-30 17:18:46 +00:00
import qualified Data.ByteString.Lazy as L
2011-03-30 17:18:46 +00:00
import qualified Data.Map as M
import qualified System.FilePath.ByteString as P
import Data.Default
2023-03-21 22:22:41 +00:00
import System.PosixCompat.Files (isRegularFile, deviceID)
#ifndef mingw32_HOST_OS
import System.PosixCompat.Files (getFdStatus)
#endif
2011-03-30 17:18:46 +00:00
import Annex.Common
import Types.Remote
2017-09-15 20:34:45 +00:00
import Types.Export
import Types.Creds
import qualified Git
import Config.Cost
import Config
import Annex.SpecialRemote.Config
2011-08-17 00:49:54 +00:00
import Remote.Helper.Special
2019-02-20 19:55:01 +00:00
import Remote.Helper.ExportImport
import Remote.Helper.Path
import Types.Import
import qualified Remote.Directory.LegacyChunked as Legacy
import Annex.CopyFile
import Annex.Content
import Annex.Perms
import Annex.UUID
import Annex.Verify
import Backend
import Types.KeySource
import Types.ProposedAccepted
import Utility.Metered
import Utility.Tmp
import Utility.InodeCache
import Utility.FileMode
import Utility.Directory.Create
import qualified Utility.RawFilePath as R
#ifndef mingw32_HOST_OS
import Utility.OpenFd
#endif
2011-03-30 17:18:46 +00:00
2011-12-31 08:11:39 +00:00
remote :: RemoteType
remote = specialRemoteType $ RemoteType
{ typename = "directory"
, enumerate = const (findSpecialRemotes "directory")
, generate = gen
, configParser = mkRemoteConfigParser
[ optionalStringParser directoryField
(FieldDesc "(required) where the special remote stores data")
, yesNoParser ignoreinodesField (Just False)
(FieldDesc "ignore inodes when importing/exporting")
]
, setup = directorySetup
, exportSupported = exportIsSupported
, importSupported = importIsSupported
add thirdPartyPopulated interface This is to support, eg a borg repo as a special remote, which is populated not by running git-annex commands, but by using borg. Then git-annex sync lists the content of the remote, learns which files are annex objects, and treats those as present in the remote. So, most of the import machinery is reused, to a new purpose. While normally importtree maintains a remote tracking branch, this does not, because the files stored in the remote are annex object files, not user-visible filenames. But, internally, a git tree is still generated, of the files on the remote that are annex objects. This tree is used by retrieveExportWithContentIdentifier, etc. As with other import/export remotes, that the tree is recorded in the export log, and gets grafted into the git-annex branch. importKey changed to be able to return Nothing, to indicate when an ImportLocation is not an annex object and so should be skipped from being included in the tree. It did not seem to make sense to have git-annex import do this, since from the user's perspective, it's not like other imports. So only git-annex sync does it. Note that, git-annex sync does not yet download objects from such remotes that are preferred content. importKeys is run with content downloading disabled, to avoid getting the content of all objects. Perhaps what's needed is for seekSyncContent to be run with these remotes, but I don't know if it will just work (in particular, it needs to avoid trying to transfer objects to them), so I skipped that for now. (Untested and unused as of yet.) This commit was sponsored by Jochen Bartl on Patreon.
2020-12-18 18:52:57 +00:00
, thirdPartyPopulated = False
}
2011-03-30 17:18:46 +00:00
directoryField :: RemoteConfigField
directoryField = Accepted "directory"
ignoreinodesField :: RemoteConfigField
ignoreinodesField = Accepted "ignoreinodes"
gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> RemoteStateHandle -> Annex (Maybe Remote)
gen r u rc gc rs = do
c <- parsedRemoteConfig remote rc
cst <- remoteCost gc c cheapRemoteCost
let chunkconfig = getChunkConfig c
cow <- liftIO newCopyCoWTried
let ii = IgnoreInodes $ fromMaybe True $
getRemoteConfigValue ignoreinodesField c
return $ Just $ specialRemote c
(storeKeyM dir chunkconfig cow)
(retrieveKeyFileM dir chunkconfig cow)
(removeKeyM dir)
(checkPresentM dir chunkconfig)
2014-12-16 19:26:13 +00:00
Remote
{ uuid = u
, cost = cst
, name = Git.repoDescribe r
, storeKey = storeKeyDummy
, retrieveKeyFile = retrieveKeyFileDummy
, retrieveKeyFileCheap = retrieveKeyFileCheapM dir chunkconfig
, retrievalSecurityPolicy = RetrievalAllKeysSecure
2014-12-16 19:26:13 +00:00
, removeKey = removeKeyDummy
, lockContent = Nothing
2014-12-16 19:26:13 +00:00
, checkPresent = checkPresentDummy
, checkPresentCheap = True
, exportActions = ExportActions
{ storeExport = storeExportM dir cow
2021-04-14 20:10:09 +00:00
, retrieveExport = retrieveExportM dir cow
, removeExport = removeExportM dir
, versionedExport = False
, checkPresentExport = checkPresentExportM dir
-- Not needed because removeExportLocation
-- auto-removes empty directories.
, removeExportDirectory = Nothing
, renameExport = Just $ renameExportM dir
}
, importActions = ImportActions
{ listImportableContents = listImportableContentsM ii dir
, importKey = Just (importKeyM ii dir)
, retrieveExportWithContentIdentifier = retrieveExportWithContentIdentifierM ii dir cow
, storeExportWithContentIdentifier = storeExportWithContentIdentifierM ii dir cow
, removeExportWithContentIdentifier = removeExportWithContentIdentifierM ii dir
-- Not needed because removeExportWithContentIdentifier
-- auto-removes empty directories.
, removeExportDirectoryWhenEmpty = Nothing
, checkPresentExportWithContentIdentifier = checkPresentExportWithContentIdentifierM ii dir
}
2014-12-16 19:26:13 +00:00
, whereisKey = Nothing
, remoteFsck = Nothing
, repairRepo = Nothing
, config = c
, getRepo = return r
2014-12-16 19:26:13 +00:00
, gitconfig = gc
, localpath = Just dir'
2014-12-16 19:26:13 +00:00
, readonly = False
, appendonly = False
, untrustworthy = False
, availability = checkPathAvailability True dir'
2014-12-16 19:26:13 +00:00
, remotetype = remote
, mkUnavailable = gen r u rc
(gc { remoteAnnexDirectory = Just "/dev/null" }) rs
, getInfo = return [("directory", dir')]
2014-12-16 19:26:13 +00:00
, claimUrl = Nothing
, checkUrl = Nothing
, remoteStateHandle = rs
2014-12-16 19:26:13 +00:00
}
where
dir = toRawFilePath dir'
dir' = fromMaybe (giveup "missing directory") (remoteAnnexDirectory gc)
directorySetup :: SetupStage -> Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID)
directorySetup _ mu _ c gc = do
u <- maybe (liftIO genUUID) return mu
2011-03-30 17:18:46 +00:00
-- verify configuration is sane
let dir = maybe (giveup "Specify directory=") fromProposedAccepted $
M.lookup directoryField c
absdir <- liftIO $ fromRawFilePath <$> absPath (toRawFilePath dir)
liftIO $ unlessM (doesDirectoryExist absdir) $
giveup $ "Directory does not exist: " ++ absdir
(c', _encsetup) <- encryptionSetup c gc
2011-03-30 17:18:46 +00:00
-- The directory is stored in git config, not in this remote's
2023-03-14 02:39:16 +00:00
-- persistent state, so it can vary between hosts.
gitConfigSpecialRemote u c' [("directory", absdir)]
return (M.delete directoryField c', u)
2011-03-30 17:18:46 +00:00
{- Locations to try to access a given Key in the directory.
- We try more than one since we used to write to different hash
- directories. -}
locations :: RawFilePath -> Key -> [RawFilePath]
locations d k = map (d P.</>) (keyPaths k)
2011-03-30 17:18:46 +00:00
{- Returns the location off a Key in the directory. If the key is
- present, returns the location that is actually used, otherwise
- returns the first, default location. -}
getLocation :: RawFilePath -> Key -> IO RawFilePath
getLocation d k = do
let locs = locations d k
fromMaybe (Prelude.head locs)
<$> firstM (doesFileExist . fromRawFilePath) locs
{- Directory where the file(s) for a key are stored. -}
storeDir :: RawFilePath -> Key -> RawFilePath
storeDir d k = P.addTrailingPathSeparator $
d P.</> hashDirLower def k P.</> keyFile k
{- Check if there is enough free disk space in the remote's directory to
- store the key. Note that the unencrypted key size is checked. -}
storeKeyM :: RawFilePath -> ChunkConfig -> CopyCoWTried -> Storer
storeKeyM d chunkconfig cow k c m =
ifM (checkDiskSpaceDirectory d k)
( do
void $ liftIO $ tryIO $ createDirectoryUnder [d] tmpdir
store
, giveup "Not enough free disk space."
)
where
store = case chunkconfig of
LegacyChunks chunksize ->
let go _k b p = liftIO $ Legacy.store
(fromRawFilePath d)
chunksize
(finalizeStoreGeneric d)
k b p
(fromRawFilePath tmpdir)
(fromRawFilePath destdir)
in byteStorer go k c m
NoChunks ->
let go _k src p = liftIO $ do
void $ fileCopier cow src tmpf p Nothing
finalizeStoreGeneric d tmpdir destdir
in fileStorer go k c m
_ ->
let go _k b p = liftIO $ do
meteredWriteFile p tmpf b
finalizeStoreGeneric d tmpdir destdir
in byteStorer go k c m
tmpdir = P.addTrailingPathSeparator $ d P.</> "tmp" P.</> kf
tmpf = fromRawFilePath tmpdir </> fromRawFilePath kf
kf = keyFile k
destdir = storeDir d k
checkDiskSpaceDirectory :: RawFilePath -> Key -> Annex Bool
checkDiskSpaceDirectory d k = do
annexdir <- fromRepo gitAnnexObjectDir
samefilesystem <- liftIO $ catchDefaultIO False $
(\a b -> deviceID a == deviceID b)
<$> R.getSymbolicLinkStatus d
<*> R.getSymbolicLinkStatus annexdir
checkDiskSpace Nothing (Just d) k 0 samefilesystem
{- Passed a temp directory that contains the files that should be placed
- in the dest directory, moves it into place. Anything already existing
- in the dest directory will be deleted. File permissions will be locked
- down. -}
finalizeStoreGeneric :: RawFilePath -> RawFilePath -> RawFilePath -> IO ()
finalizeStoreGeneric d tmp dest = do
removeDirGeneric False (fromRawFilePath d) dest'
createDirectoryUnder [d] (parentDir dest)
renameDirectory (fromRawFilePath tmp) dest'
-- may fail on some filesystems
void $ tryIO $ do
mapM_ (preventWrite . toRawFilePath) =<< dirContents dest'
preventWrite dest
where
dest' = fromRawFilePath dest
retrieveKeyFileM :: RawFilePath -> ChunkConfig -> CopyCoWTried -> Retriever
retrieveKeyFileM d (LegacyChunks _) _ = Legacy.retrieve locations d
retrieveKeyFileM d NoChunks cow = fileRetriever' $ \dest k p iv -> do
src <- liftIO $ fromRawFilePath <$> getLocation d k
void $ liftIO $ fileCopier cow src (fromRawFilePath dest) p iv
retrieveKeyFileM d _ _ = byteRetriever $ \k sink ->
sink =<< liftIO (L.readFile . fromRawFilePath =<< getLocation d k)
2011-04-17 01:41:14 +00:00
retrieveKeyFileCheapM :: RawFilePath -> ChunkConfig -> Maybe (Key -> AssociatedFile -> FilePath -> Annex ())
-- no cheap retrieval possible for chunks
retrieveKeyFileCheapM _ (UnpaddedChunks _) = Nothing
retrieveKeyFileCheapM _ (LegacyChunks _) = Nothing
#ifndef mingw32_HOST_OS
retrieveKeyFileCheapM d NoChunks = Just $ \k _af f -> liftIO $ do
file <- fromRawFilePath <$> (absPath =<< getLocation d k)
ifM (doesFileExist file)
( R.createSymbolicLink (toRawFilePath file) (toRawFilePath f)
, giveup "content file not present in remote"
)
#else
retrieveKeyFileCheapM _ _ = Nothing
#endif
removeKeyM :: RawFilePath -> Remover
toward SafeDropProof expiry checking Added Maybe POSIXTime to SafeDropProof, which gets set when the proof is based on a LockedCopy. If there are several LockedCopies, it uses the closest expiry time. That is not optimal, it may be that the proof expires based on one LockedCopy but another one has not expired. But that seems unlikely to really happen, and anyway the user can just re-run a drop if it fails due to expiry. Pass the SafeDropProof to removeKey, which is responsible for checking it for expiry in situations where that could be a problem. Which really only means in Remote.Git. Made Remote.Git check expiry when dropping from a local remote. Checking expiry when dropping from a P2P remote is not yet implemented. P2P.Protocol.remove has SafeDropProof plumbed through to it for that purpose. Fixing the remaining 2 build warnings should complete this work. Note that the use of a POSIXTime here means that if the clock gets set forward while git-annex is in the middle of a drop, it may say that dropping took too long. That seems ok. Less ok is that if the clock gets turned back a sufficient amount (eg 5 minutes), proof expiry won't be noticed. It might be better to use the Monotonic clock, but that doesn't advance when a laptop is suspended, and while there is the linux Boottime clock, that is not available on other systems. Perhaps a combination of POSIXTime and the Monotonic clock could detect laptop suspension and also detect clock being turned back? There is a potential future flag day where p2pDefaultLockContentRetentionDuration is not assumed, but is probed using the P2P protocol, and peers that don't support it can no longer produce a LockedCopy. Until that happens, when git-annex is communicating with older peers there is a risk of data loss when a ssh connection closes during LOCKCONTENT.
2024-07-04 16:23:46 +00:00
removeKeyM d _proof k = liftIO $ removeDirGeneric True
(fromRawFilePath d)
(fromRawFilePath (storeDir d k))
{- Removes the directory, which must be located under the topdir.
-
- Succeeds even on directories and contents that do not have write
2020-05-14 18:08:09 +00:00
- permission, if it's possible to turn the write bit on.
-
- If the directory does not exist, succeeds as long as the topdir does
- exist. If the topdir does not exist, fails, because in this case the
- remote is not currently accessible and probably still has the content
- we were supposed to remove from it.
-
- Empty parent directories (up to but not including the topdir)
- can also be removed. Failure to remove such a directory is not treated
- as an error.
-}
removeDirGeneric :: Bool -> FilePath -> FilePath -> IO ()
removeDirGeneric removeemptyparents topdir dir = do
void $ tryIO $ allowWrite (toRawFilePath dir)
#ifdef mingw32_HOST_OS
{- Windows needs the files inside the directory to be writable
- before it can delete them. -}
void $ tryIO $ mapM_ (allowWrite . toRawFilePath) =<< dirContents dir
#endif
2020-05-14 18:08:09 +00:00
tryNonAsync (removeDirectoryRecursive dir) >>= \case
Right () -> return ()
Left e ->
unlessM (doesDirectoryExist topdir <&&> (not <$> doesDirectoryExist dir)) $
throwM e
when removeemptyparents $ do
subdir <- relPathDirToFile (toRawFilePath topdir) (P.takeDirectory (toRawFilePath dir))
goparents (Just (P.takeDirectory subdir)) (Right ())
where
goparents _ (Left _e) = return ()
goparents Nothing _ = return ()
goparents (Just subdir) _ = do
let d = topdir </> fromRawFilePath subdir
goparents (upFrom subdir) =<< tryIO (removeDirectory d)
2011-03-30 17:18:46 +00:00
checkPresentM :: RawFilePath -> ChunkConfig -> CheckPresent
checkPresentM d (LegacyChunks _) k = Legacy.checkKey d locations k
checkPresentM d _ k = checkPresentGeneric d (locations d k)
add API for exporting Implemented so far for the directory special remote. Several remotes don't make sense to export to. Regular Git remotes, obviously, do not. Bup remotes almost certianly do not, since bup would need to be used to extract the export; same store for Ddar. Web and Bittorrent are download-only. GCrypt is always encrypted so exporting to it would be pointless. There's probably no point complicating the Hook remotes with exporting at this point. External, S3, Glacier, WebDAV, Rsync, and possibly Tahoe should be modified to support export. Thought about trying to reuse the storeKey/retrieveKeyFile/removeKey interface, rather than adding a new interface. But, it seemed better to keep it separate, to avoid a complicated interface that sometimes encrypts/chunks key/value storage and sometimes users non-key/value storage. Any common parts can be factored out. Note that storeExport is not atomic. doc/design/exporting_trees_to_special_remotes.mdwn has some things in the "resuming exports" section that bear on this decision. Basically, I don't think, at this time, that an atomic storeExport would help with resuming, because exports are not key/value storage, and we can't be sure that a partially uploaded file is the same content we're currently trying to export. Also, note that ExportLocation will always use unix path separators. This is important, because users may export from a mix of windows and unix, and it avoids complicating the API with path conversions, and ensures that in such a mix, they always use the same locations for exports. This commit was sponsored by Bruno BEAUFILS on Patreon.
2017-08-29 17:00:41 +00:00
checkPresentGeneric :: RawFilePath -> [RawFilePath] -> Annex Bool
checkPresentGeneric d ps = checkPresentGeneric' d $
liftIO $ anyM (doesFileExist . fromRawFilePath) ps
checkPresentGeneric' :: RawFilePath -> Annex Bool -> Annex Bool
checkPresentGeneric' d check = ifM check
( return True
, ifM (liftIO $ doesDirectoryExist (fromRawFilePath d))
( return False
, giveup $ "directory " ++ fromRawFilePath d ++ " is not accessible"
)
)
add API for exporting Implemented so far for the directory special remote. Several remotes don't make sense to export to. Regular Git remotes, obviously, do not. Bup remotes almost certianly do not, since bup would need to be used to extract the export; same store for Ddar. Web and Bittorrent are download-only. GCrypt is always encrypted so exporting to it would be pointless. There's probably no point complicating the Hook remotes with exporting at this point. External, S3, Glacier, WebDAV, Rsync, and possibly Tahoe should be modified to support export. Thought about trying to reuse the storeKey/retrieveKeyFile/removeKey interface, rather than adding a new interface. But, it seemed better to keep it separate, to avoid a complicated interface that sometimes encrypts/chunks key/value storage and sometimes users non-key/value storage. Any common parts can be factored out. Note that storeExport is not atomic. doc/design/exporting_trees_to_special_remotes.mdwn has some things in the "resuming exports" section that bear on this decision. Basically, I don't think, at this time, that an atomic storeExport would help with resuming, because exports are not key/value storage, and we can't be sure that a partially uploaded file is the same content we're currently trying to export. Also, note that ExportLocation will always use unix path separators. This is important, because users may export from a mix of windows and unix, and it avoids complicating the API with path conversions, and ensures that in such a mix, they always use the same locations for exports. This commit was sponsored by Bruno BEAUFILS on Patreon.
2017-08-29 17:00:41 +00:00
storeExportM :: RawFilePath -> CopyCoWTried -> FilePath -> Key -> ExportLocation -> MeterUpdate -> Annex ()
storeExportM d cow src _k loc p = do
liftIO $ createDirectoryUnder [d] (P.takeDirectory dest)
-- Write via temp file so that checkPresentGeneric will not
-- see it until it's fully stored.
viaTmp go (fromRawFilePath dest) ()
add API for exporting Implemented so far for the directory special remote. Several remotes don't make sense to export to. Regular Git remotes, obviously, do not. Bup remotes almost certianly do not, since bup would need to be used to extract the export; same store for Ddar. Web and Bittorrent are download-only. GCrypt is always encrypted so exporting to it would be pointless. There's probably no point complicating the Hook remotes with exporting at this point. External, S3, Glacier, WebDAV, Rsync, and possibly Tahoe should be modified to support export. Thought about trying to reuse the storeKey/retrieveKeyFile/removeKey interface, rather than adding a new interface. But, it seemed better to keep it separate, to avoid a complicated interface that sometimes encrypts/chunks key/value storage and sometimes users non-key/value storage. Any common parts can be factored out. Note that storeExport is not atomic. doc/design/exporting_trees_to_special_remotes.mdwn has some things in the "resuming exports" section that bear on this decision. Basically, I don't think, at this time, that an atomic storeExport would help with resuming, because exports are not key/value storage, and we can't be sure that a partially uploaded file is the same content we're currently trying to export. Also, note that ExportLocation will always use unix path separators. This is important, because users may export from a mix of windows and unix, and it avoids complicating the API with path conversions, and ensures that in such a mix, they always use the same locations for exports. This commit was sponsored by Bruno BEAUFILS on Patreon.
2017-08-29 17:00:41 +00:00
where
dest = exportPath d loc
go tmp () = void $ liftIO $ fileCopier cow src tmp p Nothing
add API for exporting Implemented so far for the directory special remote. Several remotes don't make sense to export to. Regular Git remotes, obviously, do not. Bup remotes almost certianly do not, since bup would need to be used to extract the export; same store for Ddar. Web and Bittorrent are download-only. GCrypt is always encrypted so exporting to it would be pointless. There's probably no point complicating the Hook remotes with exporting at this point. External, S3, Glacier, WebDAV, Rsync, and possibly Tahoe should be modified to support export. Thought about trying to reuse the storeKey/retrieveKeyFile/removeKey interface, rather than adding a new interface. But, it seemed better to keep it separate, to avoid a complicated interface that sometimes encrypts/chunks key/value storage and sometimes users non-key/value storage. Any common parts can be factored out. Note that storeExport is not atomic. doc/design/exporting_trees_to_special_remotes.mdwn has some things in the "resuming exports" section that bear on this decision. Basically, I don't think, at this time, that an atomic storeExport would help with resuming, because exports are not key/value storage, and we can't be sure that a partially uploaded file is the same content we're currently trying to export. Also, note that ExportLocation will always use unix path separators. This is important, because users may export from a mix of windows and unix, and it avoids complicating the API with path conversions, and ensures that in such a mix, they always use the same locations for exports. This commit was sponsored by Bruno BEAUFILS on Patreon.
2017-08-29 17:00:41 +00:00
retrieveExportM :: RawFilePath -> CopyCoWTried -> Key -> ExportLocation -> FilePath -> MeterUpdate -> Annex Verification
retrieveExportM d cow k loc dest p =
verifyKeyContentIncrementally AlwaysVerify k $ \iv ->
void $ liftIO $ fileCopier cow src dest p iv
add API for exporting Implemented so far for the directory special remote. Several remotes don't make sense to export to. Regular Git remotes, obviously, do not. Bup remotes almost certianly do not, since bup would need to be used to extract the export; same store for Ddar. Web and Bittorrent are download-only. GCrypt is always encrypted so exporting to it would be pointless. There's probably no point complicating the Hook remotes with exporting at this point. External, S3, Glacier, WebDAV, Rsync, and possibly Tahoe should be modified to support export. Thought about trying to reuse the storeKey/retrieveKeyFile/removeKey interface, rather than adding a new interface. But, it seemed better to keep it separate, to avoid a complicated interface that sometimes encrypts/chunks key/value storage and sometimes users non-key/value storage. Any common parts can be factored out. Note that storeExport is not atomic. doc/design/exporting_trees_to_special_remotes.mdwn has some things in the "resuming exports" section that bear on this decision. Basically, I don't think, at this time, that an atomic storeExport would help with resuming, because exports are not key/value storage, and we can't be sure that a partially uploaded file is the same content we're currently trying to export. Also, note that ExportLocation will always use unix path separators. This is important, because users may export from a mix of windows and unix, and it avoids complicating the API with path conversions, and ensures that in such a mix, they always use the same locations for exports. This commit was sponsored by Bruno BEAUFILS on Patreon.
2017-08-29 17:00:41 +00:00
where
src = fromRawFilePath $ exportPath d loc
add API for exporting Implemented so far for the directory special remote. Several remotes don't make sense to export to. Regular Git remotes, obviously, do not. Bup remotes almost certianly do not, since bup would need to be used to extract the export; same store for Ddar. Web and Bittorrent are download-only. GCrypt is always encrypted so exporting to it would be pointless. There's probably no point complicating the Hook remotes with exporting at this point. External, S3, Glacier, WebDAV, Rsync, and possibly Tahoe should be modified to support export. Thought about trying to reuse the storeKey/retrieveKeyFile/removeKey interface, rather than adding a new interface. But, it seemed better to keep it separate, to avoid a complicated interface that sometimes encrypts/chunks key/value storage and sometimes users non-key/value storage. Any common parts can be factored out. Note that storeExport is not atomic. doc/design/exporting_trees_to_special_remotes.mdwn has some things in the "resuming exports" section that bear on this decision. Basically, I don't think, at this time, that an atomic storeExport would help with resuming, because exports are not key/value storage, and we can't be sure that a partially uploaded file is the same content we're currently trying to export. Also, note that ExportLocation will always use unix path separators. This is important, because users may export from a mix of windows and unix, and it avoids complicating the API with path conversions, and ensures that in such a mix, they always use the same locations for exports. This commit was sponsored by Bruno BEAUFILS on Patreon.
2017-08-29 17:00:41 +00:00
removeExportM :: RawFilePath -> Key -> ExportLocation -> Annex ()
removeExportM d _k loc = liftIO $ do
removeWhenExistsWith R.removeLink src
removeExportLocation d loc
add API for exporting Implemented so far for the directory special remote. Several remotes don't make sense to export to. Regular Git remotes, obviously, do not. Bup remotes almost certianly do not, since bup would need to be used to extract the export; same store for Ddar. Web and Bittorrent are download-only. GCrypt is always encrypted so exporting to it would be pointless. There's probably no point complicating the Hook remotes with exporting at this point. External, S3, Glacier, WebDAV, Rsync, and possibly Tahoe should be modified to support export. Thought about trying to reuse the storeKey/retrieveKeyFile/removeKey interface, rather than adding a new interface. But, it seemed better to keep it separate, to avoid a complicated interface that sometimes encrypts/chunks key/value storage and sometimes users non-key/value storage. Any common parts can be factored out. Note that storeExport is not atomic. doc/design/exporting_trees_to_special_remotes.mdwn has some things in the "resuming exports" section that bear on this decision. Basically, I don't think, at this time, that an atomic storeExport would help with resuming, because exports are not key/value storage, and we can't be sure that a partially uploaded file is the same content we're currently trying to export. Also, note that ExportLocation will always use unix path separators. This is important, because users may export from a mix of windows and unix, and it avoids complicating the API with path conversions, and ensures that in such a mix, they always use the same locations for exports. This commit was sponsored by Bruno BEAUFILS on Patreon.
2017-08-29 17:00:41 +00:00
where
src = exportPath d loc
add API for exporting Implemented so far for the directory special remote. Several remotes don't make sense to export to. Regular Git remotes, obviously, do not. Bup remotes almost certianly do not, since bup would need to be used to extract the export; same store for Ddar. Web and Bittorrent are download-only. GCrypt is always encrypted so exporting to it would be pointless. There's probably no point complicating the Hook remotes with exporting at this point. External, S3, Glacier, WebDAV, Rsync, and possibly Tahoe should be modified to support export. Thought about trying to reuse the storeKey/retrieveKeyFile/removeKey interface, rather than adding a new interface. But, it seemed better to keep it separate, to avoid a complicated interface that sometimes encrypts/chunks key/value storage and sometimes users non-key/value storage. Any common parts can be factored out. Note that storeExport is not atomic. doc/design/exporting_trees_to_special_remotes.mdwn has some things in the "resuming exports" section that bear on this decision. Basically, I don't think, at this time, that an atomic storeExport would help with resuming, because exports are not key/value storage, and we can't be sure that a partially uploaded file is the same content we're currently trying to export. Also, note that ExportLocation will always use unix path separators. This is important, because users may export from a mix of windows and unix, and it avoids complicating the API with path conversions, and ensures that in such a mix, they always use the same locations for exports. This commit was sponsored by Bruno BEAUFILS on Patreon.
2017-08-29 17:00:41 +00:00
checkPresentExportM :: RawFilePath -> Key -> ExportLocation -> Annex Bool
checkPresentExportM d _k loc =
add API for exporting Implemented so far for the directory special remote. Several remotes don't make sense to export to. Regular Git remotes, obviously, do not. Bup remotes almost certianly do not, since bup would need to be used to extract the export; same store for Ddar. Web and Bittorrent are download-only. GCrypt is always encrypted so exporting to it would be pointless. There's probably no point complicating the Hook remotes with exporting at this point. External, S3, Glacier, WebDAV, Rsync, and possibly Tahoe should be modified to support export. Thought about trying to reuse the storeKey/retrieveKeyFile/removeKey interface, rather than adding a new interface. But, it seemed better to keep it separate, to avoid a complicated interface that sometimes encrypts/chunks key/value storage and sometimes users non-key/value storage. Any common parts can be factored out. Note that storeExport is not atomic. doc/design/exporting_trees_to_special_remotes.mdwn has some things in the "resuming exports" section that bear on this decision. Basically, I don't think, at this time, that an atomic storeExport would help with resuming, because exports are not key/value storage, and we can't be sure that a partially uploaded file is the same content we're currently trying to export. Also, note that ExportLocation will always use unix path separators. This is important, because users may export from a mix of windows and unix, and it avoids complicating the API with path conversions, and ensures that in such a mix, they always use the same locations for exports. This commit was sponsored by Bruno BEAUFILS on Patreon.
2017-08-29 17:00:41 +00:00
checkPresentGeneric d [exportPath d loc]
renameExportM :: RawFilePath -> Key -> ExportLocation -> ExportLocation -> Annex (Maybe ())
renameExportM d _k oldloc newloc = liftIO $ do
createDirectoryUnder [d] (P.takeDirectory dest)
renameFile (fromRawFilePath src) (fromRawFilePath dest)
removeExportLocation d oldloc
return (Just ())
add API for exporting Implemented so far for the directory special remote. Several remotes don't make sense to export to. Regular Git remotes, obviously, do not. Bup remotes almost certianly do not, since bup would need to be used to extract the export; same store for Ddar. Web and Bittorrent are download-only. GCrypt is always encrypted so exporting to it would be pointless. There's probably no point complicating the Hook remotes with exporting at this point. External, S3, Glacier, WebDAV, Rsync, and possibly Tahoe should be modified to support export. Thought about trying to reuse the storeKey/retrieveKeyFile/removeKey interface, rather than adding a new interface. But, it seemed better to keep it separate, to avoid a complicated interface that sometimes encrypts/chunks key/value storage and sometimes users non-key/value storage. Any common parts can be factored out. Note that storeExport is not atomic. doc/design/exporting_trees_to_special_remotes.mdwn has some things in the "resuming exports" section that bear on this decision. Basically, I don't think, at this time, that an atomic storeExport would help with resuming, because exports are not key/value storage, and we can't be sure that a partially uploaded file is the same content we're currently trying to export. Also, note that ExportLocation will always use unix path separators. This is important, because users may export from a mix of windows and unix, and it avoids complicating the API with path conversions, and ensures that in such a mix, they always use the same locations for exports. This commit was sponsored by Bruno BEAUFILS on Patreon.
2017-08-29 17:00:41 +00:00
where
src = exportPath d oldloc
dest = exportPath d newloc
exportPath :: RawFilePath -> ExportLocation -> RawFilePath
exportPath d loc = d P.</> fromExportLocation loc
{- Removes the ExportLocation's parent directory and its parents, so long as
- they're empty, up to but not including the topdir. -}
removeExportLocation :: RawFilePath -> ExportLocation -> IO ()
removeExportLocation topdir loc =
go (Just $ P.takeDirectory $ fromExportLocation loc) (Right ())
where
go _ (Left _e) = return ()
go Nothing _ = return ()
go (Just loc') _ =
let p = fromRawFilePath $ exportPath topdir $
mkExportLocation loc'
in go (upFrom loc') =<< tryIO (removeDirectory p)
listImportableContentsM :: IgnoreInodes -> RawFilePath -> Annex (Maybe (ImportableContentsChunkable Annex (ContentIdentifier, ByteSize)))
listImportableContentsM ii dir = liftIO $ do
l <- dirContentsRecursiveSkipping (const False) False (fromRawFilePath dir)
l' <- mapM (go . toRawFilePath) l
return $ Just $ ImportableContentsComplete $
ImportableContents (catMaybes l') []
where
go f = do
st <- R.getSymbolicLinkStatus f
mkContentIdentifier ii f st >>= \case
Nothing -> return Nothing
Just cid -> do
relf <- relPathDirToFile dir f
sz <- getFileSize' f st
return $ Just (mkImportLocation relf, (cid, sz))
newtype IgnoreInodes = IgnoreInodes Bool
-- Make a ContentIdentifier that contains the size and mtime of the file,
-- and also normally the inode, unless ignoreinodes=yes.
--
-- If the file is not a regular file, this will return Nothing.
mkContentIdentifier :: IgnoreInodes -> RawFilePath -> FileStatus -> IO (Maybe ContentIdentifier)
mkContentIdentifier (IgnoreInodes ii) f st =
liftIO $ fmap (ContentIdentifier . encodeBS . showInodeCache)
<$> if ii
then toInodeCache' noTSDelta f st 0
else toInodeCache noTSDelta f st
-- Since ignoreinodes can be changed by enableremote, and since previous
-- versions of git-annex ignored inodes by default, treat two content
-- identifiers as the same if they differ only by one having the inode
-- ignored.
guardSameContentIdentifiers :: a -> [ContentIdentifier] -> Maybe ContentIdentifier -> a
guardSameContentIdentifiers _ _ Nothing = giveup "file not found"
guardSameContentIdentifiers cont olds (Just new)
| any (new ==) olds = cont
| any (ignoreinode new ==) olds = cont
| any (\old -> new == ignoreinode old) olds = cont
| otherwise = giveup "file content has changed"
where
ignoreinode cid@(ContentIdentifier b) =
case readInodeCache (decodeBS b) of
Nothing -> cid
Just ic ->
let ic' = replaceInode 0 ic
in ContentIdentifier (encodeBS (showInodeCache ic'))
importKeyM :: IgnoreInodes -> RawFilePath -> ExportLocation -> ContentIdentifier -> ByteSize -> MeterUpdate -> Annex (Maybe Key)
importKeyM ii dir loc cid sz p = do
backend <- chooseBackend f
unsizedk <- fst <$> genKey ks p backend
let k = alterKey unsizedk $ \kd -> kd
{ keySize = keySize kd <|> Just sz }
currcid <- liftIO $ mkContentIdentifier ii absf
=<< R.getSymbolicLinkStatus absf
guardSameContentIdentifiers (return (Just k)) [cid] currcid
where
f = fromExportLocation loc
absf = dir P.</> f
ks = KeySource
{ keyFilename = f
, contentLocation = absf
, inodeCache = Nothing
}
retrieveExportWithContentIdentifierM :: IgnoreInodes -> RawFilePath -> CopyCoWTried -> ExportLocation -> [ContentIdentifier] -> FilePath -> Either Key (Annex Key) -> MeterUpdate -> Annex (Key, Verification)
retrieveExportWithContentIdentifierM ii dir cow loc cids dest gk p =
case gk of
Right mkkey -> do
go Nothing
k <- mkkey
return (k, UnVerified)
Left k -> do
v <- verifyKeyContentIncrementally DefaultVerify k go
return (k, v)
where
f = exportPath dir loc
f' = fromRawFilePath f
go iv = precheck (docopy iv)
docopy iv = ifM (liftIO $ tryCopyCoW cow f' dest p)
( postcheckcow (liftIO $ maybe noop unableIncrementalVerifier iv)
, docopynoncow iv
2021-04-14 20:10:09 +00:00
)
docopynoncow iv = do
#ifndef mingw32_HOST_OS
let open = do
-- Need a duplicate fd for the post check.
fd <- openFdWithMode f ReadOnly Nothing defaultFileFlags
dupfd <- dup fd
h <- fdToHandle fd
return (h, dupfd)
let close (h, dupfd) = do
hClose h
closeFd dupfd
bracketIO open close $ \(h, dupfd) -> do
#else
let open = openBinaryFile f' ReadMode
let close = hClose
2020-07-02 15:35:05 +00:00
bracketIO open close $ \h -> do
#endif
liftIO $ fileContentCopier h dest p iv
#ifndef mingw32_HOST_OS
postchecknoncow dupfd (return ())
#else
postchecknoncow (return ())
#endif
-- Check before copy, to avoid expensive copy of wrong file
-- content.
precheck cont = guardSameContentIdentifiers cont cids
=<< liftIO . mkContentIdentifier ii f
=<< liftIO (R.getSymbolicLinkStatus f)
-- Check after copy, in case the file was changed while it was
-- being copied.
--
-- When possible (not on Windows), check the same handle
-- that the file was copied from. Avoids some race cases where
-- the file is modified while it's copied but then gets restored
-- to the original content afterwards.
--
-- This does not guard against every possible race, but neither
-- can InodeCaches detect every possible modification to a file.
-- It's probably as good or better than git's handling of similar
-- situations with files being modified while it's updating the
-- working tree for a merge.
#ifndef mingw32_HOST_OS
2021-04-14 20:10:09 +00:00
postchecknoncow fd cont = do
#else
2021-04-14 20:10:09 +00:00
postchecknoncow cont = do
#endif
currcid <- liftIO $ mkContentIdentifier ii f
#ifndef mingw32_HOST_OS
=<< getFdStatus fd
#else
=<< R.getSymbolicLinkStatus f
#endif
guardSameContentIdentifiers cont cids currcid
2021-04-14 20:10:09 +00:00
-- When copy-on-write was done, cannot check the handle that was
-- copied from, but such a copy should run very fast, so
-- it's very unlikely that the file changed after precheck,
-- the modified version was copied CoW, and then the file was
-- restored to the original content before this check.
postcheckcow cont = do
currcid <- liftIO $ mkContentIdentifier ii f
=<< R.getSymbolicLinkStatus f
guardSameContentIdentifiers cont cids currcid
2021-04-14 20:10:09 +00:00
storeExportWithContentIdentifierM :: IgnoreInodes -> RawFilePath -> CopyCoWTried -> FilePath -> Key -> ExportLocation -> [ContentIdentifier] -> MeterUpdate -> Annex ContentIdentifier
storeExportWithContentIdentifierM ii dir cow src _k loc overwritablecids p = do
liftIO $ createDirectoryUnder [dir] (toRawFilePath destdir)
withTmpFileIn destdir template $ \tmpf tmph -> do
liftIO $ hClose tmph
void $ liftIO $ fileCopier cow src tmpf p Nothing
let tmpf' = toRawFilePath tmpf
resetAnnexFilePerm tmpf'
liftIO (R.getSymbolicLinkStatus tmpf') >>= liftIO . mkContentIdentifier ii tmpf' >>= \case
2020-05-15 16:17:15 +00:00
Nothing -> giveup "unable to generate content identifier"
Just newcid -> do
checkExportContent ii dir loc
overwritablecids
2020-05-15 16:17:15 +00:00
(giveup "unsafe to overwrite file")
(const $ liftIO $ R.rename tmpf' dest)
2020-05-15 16:17:15 +00:00
return newcid
where
dest = exportPath dir loc
(destdir, base) = splitFileName (fromRawFilePath dest)
template = relatedTemplate (base ++ ".tmp")
removeExportWithContentIdentifierM :: IgnoreInodes -> RawFilePath -> Key -> ExportLocation -> [ContentIdentifier] -> Annex ()
removeExportWithContentIdentifierM ii dir k loc removeablecids =
checkExportContent ii dir loc removeablecids (giveup "unsafe to remove modified file") $ \case
DoesNotExist -> return ()
KnownContentIdentifier -> removeExportM dir k loc
checkPresentExportWithContentIdentifierM :: IgnoreInodes -> RawFilePath -> Key -> ExportLocation -> [ContentIdentifier] -> Annex Bool
checkPresentExportWithContentIdentifierM ii dir _k loc knowncids =
checkPresentGeneric' dir $
checkExportContent ii dir loc knowncids (return False) $ \case
DoesNotExist -> return False
KnownContentIdentifier -> return True
data CheckResult = DoesNotExist | KnownContentIdentifier
-- Checks if the content at an ExportLocation is in the knowncids,
-- and only runs the callback that modifies it if it's safe to do so.
--
-- This should avoid races to the extent possible. However,
-- if something has the file open for write, it could write to the handle
-- after the callback has overwritten or deleted it, and its write would
-- be lost, and we don't need to detect that.
-- (In similar situations, git doesn't either!)
--
-- It follows that if something is written to the destination file
-- shortly before, it's acceptable to run the callback anyway, as that's
-- nearly indistinguishable from the above case.
--
-- So, it suffices to check if the destination file's current
-- content is known, and immediately run the callback.
checkExportContent :: IgnoreInodes -> RawFilePath -> ExportLocation -> [ContentIdentifier] -> Annex a -> (CheckResult -> Annex a) -> Annex a
checkExportContent ii dir loc knowncids unsafe callback =
tryWhenExists (liftIO $ R.getSymbolicLinkStatus dest) >>= \case
Just destst
2020-05-15 16:17:15 +00:00
| not (isRegularFile destst) -> unsafe
| otherwise -> catchDefaultIO Nothing (liftIO $ mkContentIdentifier ii dest destst) >>= \case
Just destcid
| destcid `elem` knowncids -> callback KnownContentIdentifier
-- dest exists with other content
2020-05-15 16:17:15 +00:00
| otherwise -> unsafe
-- should never happen
2020-05-15 16:17:15 +00:00
Nothing -> unsafe
-- dest does not exist
Nothing -> callback DoesNotExist
where
dest = exportPath dir loc