Added ways to configure rsync options to be used only when uploading or downloading from a remote. Useful to eg limit upload bandwidth.

Remote/Git.hs

@@ -310,7 +310,7 @@ copyFromRemote r key file dest _p = copyFromRemote' r key file dest
 copyFromRemote' :: Remote -> Key -> AssociatedFile -> FilePath -> Annex Bool
 copyFromRemote' r key file dest
 	| not $ Git.repoIsUrl (repo r) = guardUsable (repo r) False $ do
-		let params = Ssh.rsyncParams r
+		let params = Ssh.rsyncParams r Download
 		u <- getUUID
 		-- run copy from perspective of remote
 		liftIO $ onLocal (repo r) $ do
@@ -409,7 +409,7 @@ copyToRemote r key file p
 		-- the remote's Annex, but it needs access to the current
 		-- Annex monad's state.
 		checksuccessio <- Annex.withCurrentState checksuccess
-		let params = Ssh.rsyncParams r
+		let params = Ssh.rsyncParams r Upload
 		u <- getUUID
 		-- run copy from perspective of remote
 		liftIO $ onLocal (repo r) $ ifM (Annex.Content.inAnnex key)

Remote/Helper/Ssh.hs

@@ -122,7 +122,7 @@ rsyncParamsRemote direct r direction key file afile = do
 		fields
 	-- Convert the ssh command into rsync command line.
 	let eparam = rsyncShell (Param shellcmd:shellparams)
-	let o = rsyncParams r
+	let o = rsyncParams r direction
 	return $ if direction == Download
 		then o ++ rsyncopts eparam dummy (File file)
 		else o ++ rsyncopts eparam (File file) dummy
@@ -140,7 +140,11 @@ rsyncParamsRemote direct r direction key file afile = do
 	dummy = Param "dummy:"
 
 -- --inplace to resume partial files
-rsyncParams :: Remote -> [CommandParam]
-rsyncParams r = Params "--progress --inplace" :
-	map Param (remoteAnnexRsyncOptions $ gitconfig r)
-
+rsyncParams :: Remote -> Direction -> [CommandParam]
+rsyncParams r direction = Params "--progress --inplace" :
+	map Param (remoteAnnexRsyncOptions gc ++ dps)
+  where
+	dps
+		| direction == Download = remoteAnnexRsyncDownloadOptions gc
+		| otherwise = remoteAnnexRsyncUploadOptions gc
+	gc = gitconfig r

Remote/Rsync.hs

@@ -41,12 +41,15 @@ import Utility.Rsync
 import Utility.CopyFile
 import Utility.Metered
 import Annex.Perms
+import Logs.Transfer
 
 type RsyncUrl = String
 
 data RsyncOpts = RsyncOpts
 	{ rsyncUrl :: RsyncUrl
 	, rsyncOptions :: [CommandParam]
+	, rsyncUploadOptions :: [CommandParam]
+	, rsyncDownloadOptions :: [CommandParam]
 	, rsyncShellEscape :: Bool
 }
 
@@ -93,10 +96,16 @@ gen r u c gc = do
 			}
 
 genRsyncOpts :: RemoteConfig -> RemoteGitConfig -> [CommandParam] -> RsyncUrl -> RsyncOpts
-genRsyncOpts c gc transport url = RsyncOpts url (transport ++ opts) escape
+genRsyncOpts c gc transport url = RsyncOpts
+	{ rsyncUrl = url
+	, rsyncOptions = opts []
+	, rsyncUploadOptions = transport ++ opts (remoteAnnexRsyncUploadOptions gc)
+	, rsyncDownloadOptions = transport ++ opts (remoteAnnexRsyncDownloadOptions gc)
+	, rsyncShellEscape = M.lookup "shellescape" c /= Just "no"
+	}
   where
-	opts = map Param $ filter safe $ remoteAnnexRsyncOptions gc
-	escape = M.lookup "shellescape" c /= Just "no"
+	opts specificopts = map Param $ filter safe $
+		remoteAnnexRsyncOptions gc ++ specificopts
 	safe opt
 		-- Don't allow user to pass --delete to rsync;
 		-- that could cause it to delete other keys
@@ -257,7 +266,7 @@ withRsyncScratchDir a = do
 
 rsyncRetrieve :: RsyncOpts -> Key -> FilePath -> Maybe MeterUpdate -> Annex Bool
 rsyncRetrieve o k dest callback =
-	showResumable $ untilTrue (rsyncUrls o k) $ \u -> rsyncRemote o callback
+	showResumable $ untilTrue (rsyncUrls o k) $ \u -> rsyncRemote Download o callback
 		-- use inplace when retrieving to support resuming
 		[ Param "--inplace"
 		, Param u
@@ -272,13 +281,15 @@ showResumable a = ifM a
 		return False
 	)
 
-rsyncRemote :: RsyncOpts -> Maybe MeterUpdate -> [CommandParam] -> Annex Bool
-rsyncRemote o callback params = do
+rsyncRemote :: Direction -> RsyncOpts -> Maybe MeterUpdate -> [CommandParam] -> Annex Bool
+rsyncRemote direction o callback params = do
 	showOutput -- make way for progress bar
-	liftIO $ (maybe rsync rsyncProgress callback) ps
+	liftIO $ (maybe rsync rsyncProgress callback) $
+		opts ++ [Params "--progress"] ++ params
   where
-	defaultParams = [Params "--progress"]
-	ps = rsyncOptions o ++ defaultParams ++ params
+	opts
+		| direction == Download = rsyncDownloadOptions o
+		| otherwise = rsyncUploadOptions o
 
 {- To send a single key is slightly tricky; need to build up a temporary
  - directory structure to pass to rsync so it can create the hash
@@ -301,7 +312,7 @@ rsyncSend o callback k canrename src = withRsyncScratchDir $ \tmp -> do
 		else createLinkOrCopy src dest
 	ps <- sendParams
 	if ok
-		then showResumable $ rsyncRemote o (Just callback) $ ps ++
+		then showResumable $ rsyncRemote Upload o (Just callback) $ ps ++
 			[ Param "--recursive"
 			, partialParams
 			-- tmp/ to send contents of tmp dir

Types/GitConfig.hs

@@ -115,6 +115,8 @@ data RemoteGitConfig = RemoteGitConfig
 	 - including special remotes. -}
 	, remoteAnnexSshOptions :: [String]
 	, remoteAnnexRsyncOptions :: [String]
+	, remoteAnnexRsyncUploadOptions :: [String]
+	, remoteAnnexRsyncDownloadOptions :: [String]
 	, remoteAnnexRsyncTransport :: [String]
 	, remoteAnnexGnupgOptions :: [String]
 	, remoteAnnexRsyncUrl :: Maybe String
@@ -144,6 +146,8 @@ extractRemoteGitConfig r remotename = RemoteGitConfig
 
 	, remoteAnnexSshOptions = getoptions "ssh-options"
 	, remoteAnnexRsyncOptions = getoptions "rsync-options"
+	, remoteAnnexRsyncDownloadOptions = getoptions "rsync-download-options"
+	, remoteAnnexRsyncUploadOptions = getoptions "rsync-upload-options"
 	, remoteAnnexRsyncTransport = getoptions "rsync-transport"
 	, remoteAnnexGnupgOptions = getoptions "gnupg-options"
 	, remoteAnnexRsyncUrl = notempty $ getmaybe "rsyncurl"

debian/changelog

@@ -12,6 +12,8 @@ git-annex (5.20140128) UNRELEASED; urgency=medium
   * sync --content: Re-pull from remotes after downloading content,
     since that can take a while and other changes may be pushed in the
     meantime.
+  * Added ways to configure rsync options to be used only when uploading
+    or downloading from a remote. Useful to eg limit upload bandwidth.
 
  -- Joey Hess <joeyh@debian.org>  Tue, 28 Jan 2014 13:57:19 -0400
 

doc/git-annex.mdwn

@@ -1379,6 +1379,21 @@ Here are all the supported configuration settings.
   to or from this remote. For example, to force ipv6, and limit
   the bandwidth to 100Kbyte/s, set it to `-6 --bwlimit 100`
 
+* `remote.<name>.annex-rsync-upload-options`
+
+  Options to use when using rsync to upload a file to a remote.
+
+  These options are passed after other applicable rsync options,
+  so can be used to override them. For example, to limit upload bandwidth
+  to 10Kbye/s, set `--bwlimit 10`.
+
+* `remote.<name>.annex-rsync-download-options`
+
+  Options to use when using rsync to download a file from a remote.
+
+  These options are passed after other applicable rsync options,
+  so can be used to override them.
+
 * `remote.<name>.annex-rsync-transport`
 
   The remote shell to use to connect to the rsync remote. Possible
@@ -1402,11 +1417,12 @@ Here are all the supported configuration settings.
   precedence over the default GnuPG configuration, which is otherwise
   used.)
 
-* `annex.ssh-options`, `annex.rsync-options`, `annex.bup-split-options`,
-  `annex.gnupg-options`
+* `annex.ssh-options`, `annex.rsync-options`,
+  `annex.rsync-upload-options`, `annex.rsync-download-options`,
+  `annex.bup-split-options`, `annex.gnupg-options`
 
-  Default ssh, rsync, wget/curl, bup, and GnuPG options to use if a
-  remote does not have specific options.
+  Default options to use if a remote does not have more specific options
+  as described above.
 
 * `annex.web-options`
 

doc/todo/separate_rsync_bwlimit_options_for_upload_and_download.mdwn

@@ -1,2 +1,4 @@
 The bandwidth for upload and download are often different.  It would be useful to have different settings for upload and download limits.
 As it is, I have to keep changing annex-rsync-options options between uploads and downloads.
+
+> [[done]] --[[Joey]]