support chunking for all external special remotes!

Removing code and at the same time adding great features, including
upload/download resuming.

This commit was sponsored by Romain Lenglet.

Remote/External.hs

@@ -15,14 +15,12 @@ import Types.CleanupActions
 import qualified Git
 import Config
 import Remote.Helper.Special
-import Remote.Helper.Encryptable
+import Remote.Helper.ChunkedEncryptable
-import Crypto
 import Utility.Metered
 import Logs.Transfer
 import Logs.PreferredContent.Raw
 import Logs.RemoteState
 import Config.Cost
-import Annex.Content
 import Annex.UUID
 import Annex.Exception
 import Creds
@@ -30,7 +28,6 @@ import Creds
 import Control.Concurrent.STM
 import System.Log.Logger (debugM)
 import qualified Data.Map as M
-import qualified Data.ByteString.Lazy as L
 
 remote :: RemoteType
 remote = RemoteType {
@@ -46,15 +43,15 @@ gen r u c gc = do
 	Annex.addCleanup (RemoteCleanup u) $ stopExternal external
 	cst <- getCost external r gc
 	avail <- getAvailability external r gc
-	return $ Just $ encryptableRemote c
+	return $ Just $ chunkedEncryptableRemote c
-		(storeEncrypted external $ getGpgEncParams (c,gc))
+		(simplyPrepare $ store external)
-		(retrieveEncrypted external)
+		(simplyPrepare $ retrieve external)
 		Remote {
 			uuid = u,
 			cost = cst,
 			name = Git.repoDescribe r,
-			storeKey = store external,
+			storeKey = storeKeyDummy,
-			retrieveKeyFile = retrieve external,
+			retrieveKeyFile = retreiveKeyFileDummy,
 			retrieveKeyFileCheap = \_ _ -> return False,
 			removeKey = remove external,
 			hasKey = checkPresent external,
@@ -90,25 +87,8 @@ externalSetup mu _ c = do
 	gitConfigSpecialRemote u c'' "externaltype" externaltype
 	return (c'', u)
 
-store :: External -> Key -> AssociatedFile -> MeterUpdate -> Annex Bool
+store :: External -> Storer
-store external k _f p = sendAnnex k rollback $ \f ->
+store external = fileStorer $ \k f p ->
-	metered (Just p) k $
-		storeHelper external k f
-  where
-	rollback = void $ remove external k
-
-storeEncrypted :: External -> [CommandParam] -> (Cipher, Key) -> Key -> MeterUpdate -> Annex Bool
-storeEncrypted external gpgOpts (cipher, enck) k p = withTmp enck $ \tmp ->
-	sendAnnex k rollback $ \src -> do
-		metered (Just p) k $ \meterupdate -> do
-			liftIO $ encrypt gpgOpts cipher (feedFile src) $
-				readBytes $ L.writeFile tmp
-			storeHelper external enck tmp meterupdate
-  where
-	rollback = void $ remove external enck
-
-storeHelper :: External -> Key -> FilePath -> MeterUpdate -> Annex Bool
-storeHelper external k f p = safely $
 	handleRequest external (TRANSFER Upload k f) (Just p) $ \resp ->
 		case resp of
 			TRANSFER_SUCCESS Upload k' | k == k' ->
@@ -119,31 +99,15 @@ storeHelper external k f p = safely $
 					return False
 			_ -> Nothing
 
-retrieve :: External -> Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex Bool
+retrieve :: External -> Retriever
-retrieve external k _f d p = metered (Just p) k $
+retrieve external = fileRetriever $ \d k p -> 
-	retrieveHelper external k d
-
-retrieveEncrypted :: External -> (Cipher, Key) -> Key -> FilePath -> MeterUpdate -> Annex Bool
-retrieveEncrypted external (cipher, enck) k f p = withTmp enck $ \tmp ->
-	metered (Just p) k $ \meterupdate -> 
-		ifM (retrieveHelper external enck tmp meterupdate)
-			( liftIO $ catchBoolIO $ do
-				decrypt cipher (feedFile tmp) $
-					readBytes $ L.writeFile f
-				return True
-			, return False
-			)
-
-retrieveHelper :: External -> Key -> FilePath -> MeterUpdate -> Annex Bool
-retrieveHelper external k d p = safely $
 	handleRequest external (TRANSFER Download k d) (Just p) $ \resp ->
 		case resp of
 			TRANSFER_SUCCESS Download k'
-				| k == k' -> Just $ return True
+				| k == k' -> Just $ return ()
 			TRANSFER_FAILURE Download k' errmsg
 				| k == k' -> Just $ do
-					warning errmsg
+					error errmsg
-					return False
 			_ -> Nothing
 
 remove :: External -> Key -> Annex Bool

Remote/Helper/Chunked.hs

@@ -295,7 +295,7 @@ retrieveChunks retriever u chunkconfig encryptor basek dest basep sink
 		return True
 	tosink h _ (FileContent f) = liftIO $ do
 		sink h Nothing =<< L.readFile f
-		nukeFile h
+		nukeFile f
 		return True
 
 {- Can resume when the chunk's offset is at or before the end of

debian/changelog

@@ -1,6 +1,7 @@
 git-annex (5.20140718) UNRELEASED; urgency=medium
 
-  * New chunk= option to chunk files stored in directory remotes.
+  * New chunk= option to chunk files stored in special remotes.
+    Currently supported by: directory, and all external special remotes.
   * Partially transferred files are automatically resumed when using
     chunked remotes!
   * The old chunksize= option is deprecated. Do not use for new remotes.