From 4d3a464e83014669553af8e3f7ce57bc66034eb1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Sep 2017 14:08:00 -0400 Subject: [PATCH] export to webdav This basically works, but there's a bug when renaming a file that leaves a .git-annex-temp-content-key file in the webdav store, that never gets cleaned up. Also, exporting files with spaces to box.com seems to fail; perhaps it does not support it? This commit was supported by the NSF-funded DataLad project. --- CHANGELOG | 2 +- Command/Export.hs | 4 +- Remote/WebDAV.hs | 96 ++++++++++++++----- Remote/WebDAV/DavLocation.hs | 7 ++ doc/special_remotes/webdav.mdwn | 4 + .../using_box.com_as_a_special_remote.mdwn | 63 ++++++------ 6 files changed, 120 insertions(+), 56 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 4365ed9f9f..e885d42f82 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -4,7 +4,7 @@ git-annex (6.20170819) UNRELEASED; urgency=medium exports of trees to special remotes. * Use git-annex initremote with exporttree=yes to set up a special remote for use by git-annex export. - * Implemented export to directory and S3 special remotes. + * Implemented export to directory, S3, and webdav special remotes. * External special remote protocol extended to support export. * Support building with feed-1.0, while still supporting older versions. * init: Display an additional message when it detects a filesystem that diff --git a/Command/Export.hs b/Command/Export.hs index d2ba53dd23..52355e69d6 100644 --- a/Command/Export.hs +++ b/Command/Export.hs @@ -304,7 +304,9 @@ performRename r db ek src dest = do ( next $ cleanupRename db ek src dest -- In case the special remote does not support renaming, -- unexport the src instead. - , performUnexport r db [ek] src + , do + warning "rename failed; deleting instead" + performUnexport r db [ek] src ) cleanupRename :: ExportHandle -> ExportKey -> ExportLocation -> ExportLocation -> CommandCleanup diff --git a/Remote/WebDAV.hs b/Remote/WebDAV.hs index 4cc3c92e03..04eb35cef7 100644 --- a/Remote/WebDAV.hs +++ b/Remote/WebDAV.hs @@ -1,6 +1,6 @@ {- WebDAV remotes. - - - Copyright 2012-2014 Joey Hess + - Copyright 2012-2017 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} @@ -15,7 +15,7 @@ import qualified Data.Map as M import qualified Data.ByteString.Lazy as L import qualified Data.ByteString.UTF8 as B8 import qualified Data.ByteString.Lazy.UTF8 as L8 -import Network.HTTP.Client (HttpException(..)) +import Network.HTTP.Client (HttpException(..), RequestBody) import Network.HTTP.Types import System.IO.Error import Control.Monad.Catch @@ -46,7 +46,7 @@ remote = RemoteType , enumerate = const (findSpecialRemotes "webdav") , generate = gen , setup = webdavSetup - , exportSupported = exportUnsupported + , exportSupported = exportIsSupported } gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remote) @@ -70,7 +70,13 @@ gen r u c gc = new <$> remoteCost gc expensiveRemoteCost , lockContent = Nothing , checkPresent = checkPresentDummy , checkPresentCheap = False - , exportActions = exportUnsupported + , exportActions = ExportActions + { storeExport = storeExportDav this + , retrieveExport = retrieveExportDav this + , removeExport = removeExportDav this + , checkPresentExport = checkPresentExportDav this + , renameExport = renameExportDav this + } , whereisKey = Nothing , remoteFsck = Nothing , repairRepo = Nothing @@ -114,17 +120,21 @@ store (LegacyChunks chunksize) (Just dav) = fileStorer $ \k f p -> liftIO $ store _ (Just dav) = httpStorer $ \k reqbody -> liftIO $ goDAV dav $ do let tmp = keyTmpLocation k let dest = keyLocation k + storeHelper dav tmp dest reqbody + return True + +storeHelper :: DavHandle -> DavLocation -> DavLocation -> RequestBody -> DAVT IO () +storeHelper dav tmp dest reqbody = do void $ mkColRecursive tmpDir inLocation tmp $ putContentM' (contentType, reqbody) - finalizeStore (baseURL dav) tmp dest - return True + finalizeStore dav tmp dest -finalizeStore :: URLString -> DavLocation -> DavLocation -> DAVT IO () -finalizeStore baseurl tmp dest = do +finalizeStore :: DavHandle -> DavLocation -> DavLocation -> DAVT IO () +finalizeStore dav tmp dest = do inLocation dest $ void $ safely $ delContentM maybe noop (void . mkColRecursive) (locationParent dest) - moveDAV baseurl tmp dest + moveDAV (baseURL dav) tmp dest retrieveCheap :: Key -> AssociatedFile -> FilePath -> Annex Bool retrieveCheap _ _ _ = return False @@ -133,26 +143,29 @@ retrieve :: ChunkConfig -> Maybe DavHandle -> Retriever retrieve _ Nothing = giveup "unable to connect" retrieve (LegacyChunks _) (Just dav) = retrieveLegacyChunked dav retrieve _ (Just dav) = fileRetriever $ \d k p -> liftIO $ - goDAV dav $ - inLocation (keyLocation k) $ - withContentM $ - httpBodyRetriever d p + goDAV dav $ retrieveHelper (keyLocation k) d p + +retrieveHelper :: DavLocation -> FilePath -> MeterUpdate -> DAVT IO () +retrieveHelper loc d p = inLocation loc $ + withContentM $ httpBodyRetriever d p remove :: Maybe DavHandle -> Remover remove Nothing _ = return False -remove (Just dav) k = liftIO $ do +remove (Just dav) k = liftIO $ goDAV dav $ -- Delete the key's whole directory, including any -- legacy chunked files, etc, in a single action. - let d = keyDir k - goDAV dav $ do - v <- safely $ inLocation d delContentM - case v of - Just _ -> return True - Nothing -> do - v' <- existsDAV d - case v' of - Right False -> return True - _ -> return False + removeHelper (keyDir k) + +removeHelper :: DavLocation -> DAVT IO Bool +removeHelper d = do + v <- safely $ inLocation d delContentM + case v of + Just _ -> return True + Nothing -> do + v' <- existsDAV d + case v' of + Right False -> return True + _ -> return False checkKey :: Remote -> ChunkConfig -> Maybe DavHandle -> CheckPresent checkKey r _ Nothing _ = giveup $ name r ++ " not configured" @@ -165,6 +178,38 @@ checkKey r chunkconfig (Just dav) k = do existsDAV (keyLocation k) either giveup return v +storeExportDav :: Remote -> FilePath -> Key -> ExportLocation -> MeterUpdate -> Annex Bool +storeExportDav r f _k loc p = runExport r $ \dav -> do + reqbody <- liftIO $ httpBodyStorer f p + storeHelper dav (exportTmpLocation loc) (exportLocation loc) reqbody + return True + +retrieveExportDav :: Remote -> Key -> ExportLocation -> FilePath -> MeterUpdate -> Annex Bool +retrieveExportDav r _k loc d p = runExport r $ \_dav -> do + retrieveHelper (exportLocation loc) d p + return True + +removeExportDav :: Remote -> Key -> ExportLocation -> Annex Bool +removeExportDav r _k loc = runExport r $ \_dav -> + removeHelper (exportLocation loc) + +checkPresentExportDav :: Remote -> Key -> ExportLocation -> Annex Bool +checkPresentExportDav r _k loc = withDAVHandle r $ \mh -> case mh of + Nothing -> giveup $ name r ++ " not configured" + Just h -> liftIO $ do + v <- goDAV h $ existsDAV (exportLocation loc) + either giveup return v + +renameExportDav :: Remote -> Key -> ExportLocation -> ExportLocation -> Annex Bool +renameExportDav r _k src dest = runExport r $ \dav -> do + moveDAV (baseURL dav) (exportLocation src) (exportLocation dest) + return True + +runExport :: Remote -> (DavHandle -> DAVT IO Bool) -> Annex Bool +runExport r a = withDAVHandle r $ \mh -> case mh of + Nothing -> return False + Just h -> fromMaybe False <$> liftIO (goDAV h $ safely (a h)) + configUrl :: Remote -> Maybe URLString configUrl r = fixup <$> M.lookup "url" (config r) where @@ -278,7 +323,6 @@ existsDAV l = inLocation l check `catchNonAsync` (\e -> return (Left $ show e)) (const $ ispresent False) ispresent = return . Right --- Ignores any exceptions when performing a DAV action. safely :: DAVT IO a -> DAVT IO (Maybe a) safely = eitherToMaybe <$$> tryNonAsync @@ -351,7 +395,7 @@ storeLegacyChunked chunksize k dav b = storer locs = Legacy.storeChunked chunksize locs storehttp b recorder l s = storehttp l (L8.fromString s) finalizer tmp' dest' = goDAV dav $ - finalizeStore (baseURL dav) tmp' (fromJust $ locationParent dest') + finalizeStore dav tmp' (fromJust $ locationParent dest') tmp = addTrailingPathSeparator $ keyTmpLocation k dest = keyLocation k diff --git a/Remote/WebDAV/DavLocation.hs b/Remote/WebDAV/DavLocation.hs index daa669de11..82a3739d00 100644 --- a/Remote/WebDAV/DavLocation.hs +++ b/Remote/WebDAV/DavLocation.hs @@ -11,6 +11,7 @@ module Remote.WebDAV.DavLocation where import Types +import Types.Remote (ExportLocation(..)) import Annex.Locations import Utility.Url (URLString) #ifdef mingw32_HOST_OS @@ -46,6 +47,12 @@ keyLocation k = keyDir k ++ keyFile k keyTmpLocation :: Key -> DavLocation keyTmpLocation = tmpLocation . keyFile +exportLocation :: ExportLocation -> DavLocation +exportLocation (ExportLocation f) = f + +exportTmpLocation :: ExportLocation -> DavLocation +exportTmpLocation (ExportLocation f) = tmpLocation f + tmpLocation :: FilePath -> DavLocation tmpLocation f = tmpDir f diff --git a/doc/special_remotes/webdav.mdwn b/doc/special_remotes/webdav.mdwn index 100de8c20b..27bd38579d 100644 --- a/doc/special_remotes/webdav.mdwn +++ b/doc/special_remotes/webdav.mdwn @@ -29,6 +29,10 @@ the webdav remote. be created as needed. Use of a https URL is strongly encouraged, since HTTP basic authentication is used. +* `exporttree` - Set to "yes" to make this special remote usable + by [[git-annex-export]]. It will not be usable as a general-purpose + special remote. + * `chunk` - Enables [[chunking]] when storing large files. * `chunksize` - Deprecated version of chunk parameter above. diff --git a/doc/tips/using_box.com_as_a_special_remote.mdwn b/doc/tips/using_box.com_as_a_special_remote.mdwn index 2edd200b1e..80fa5c083f 100644 --- a/doc/tips/using_box.com_as_a_special_remote.mdwn +++ b/doc/tips/using_box.com_as_a_special_remote.mdwn @@ -1,14 +1,42 @@ -[Box.com](http://box.com/) is a file storage service, currently notable -for providing 50 gb of free storage if you sign up with its Android client. -(Or a few gb free otherwise.) +[Box.com](http://box.com/) is a file storage service. git-annex can use Box as a [[special remote|special_remotes]]. -Recent versions of git-annex make this very easy to set up: +Recent versions of git-annex make this very easy to set up +and use. - WEBDAV_USERNAME=you@example.com WEBDAV_PASSWORD=xxxxxxx git annex initremote box.com type=webdav url=https://dav.box.com/dav/git-annex chunk=50mb encryption=shared +## git-annex setup -Note the use of [[chunking]]; Box has a 100 mb maximum file size, and this -breaks up large files into chunks before that limit is reached. +Create the special remote, in your git-annex repository. +** This example is non-encrypted; fill in your gpg key ID for a securely +encrypted special remote! ** + + WEBDAV_USERNAME=you@example.com WEBDAV_PASSWORD=xxxxxxx git annex initremote box.com type=webdav url=https://dav.box.com/dav/git-annex chunk=50mb encryption=none + +Note the use of [[chunking]]. Box has a limit on the maximum size of file +that can be stored there (currently 256 MB). git-annex can break up large +files into chunks to avoid the size limit. This needs git-annex version +3.20120303 or newer, which adds support for chunking. + +Now git-annex can copy files to box.com, get files from it, etc, just like +with any other special remote. + + % git annex copy bigfile --to box.com + bigfile (to box.com...) ok + % git annex drop bigfile + bigfile (checking box.com...) ok + % git annex get bigfile + bigfile (from box.com...) ok + +## exporting trees + +By default, files stored in Box will show up there named +by their git-annex key, not the original filename. If the filenames +are important, you can run `git annex initremote` with an additional +parameter "exporttree=yes", and then use [[git-annex-export]] to publish +a tree of files to Box. + +Note that chunking can't be used when exporting a tree of files, +so Box's 250 mb limit will prevent exporting larger files. # old davfs2 method @@ -48,24 +76,3 @@ using the webdav special remote. * Now you should be able to mount Box, as a non-root user: mount /media/box.com - -## git-annex setup - -You need git-annex version 3.20120303 or newer, which adds support for chunking -files larger than Box's 100 mb limit. - -Create the special remote, in your git-annex repository. -** This example is non-encrypted; fill in your gpg key ID for a securely -encrypted special remote! ** - - git annex initremote box.com type=directory directory=/media/box.com chunk=2mb encryption=none - -Now git-annex can copy files to box.com, get files from it, etc, just like -with any other special remote. - - % git annex copy bigfile --to box.com - bigfile (to box.com...) ok - % git annex drop bigfile - bigfile (checking box.com...) ok - % git annex get bigfile - bigfile (from box.com...) ok