max-git-bundles config for git-remote-annex

CmdLine/GitRemoteAnnex.hs

@@ -203,7 +203,6 @@ push st rmt ls = do
 		then pushEmpty st rmt
 		else if any forcedPush refspecs
 			then fullPush st rmt (M.keys trackingrefs)
-			-- TODO: support max-bundles config
 			else incrementalPush st rmt
 				(trackingRefs st) trackingrefs
 	if ok
@@ -276,6 +275,10 @@ fullPush :: State -> Remote -> [Ref] -> Annex (Bool, State)
 fullPush st rmt refs = guardPush st $ do
 	oldmanifest <- maybe (downloadManifestWhenPresent rmt) pure
 		(manifestCache st)
+	fullPush' oldmanifest st rmt refs
+
+fullPush' :: Manifest -> State -> Remote -> [Ref] -> Annex (Bool, State)
+fullPush' oldmanifest st rmt refs =do
 	let bs = map Git.Bundle.fullBundleSpec refs
 	bundlekey <- generateAndUploadGitBundle rmt bs oldmanifest
 	uploadManifest rmt (mkManifest [bundlekey] [])
@@ -295,12 +298,17 @@ guardPush st a = catchNonAsync a $ \ex -> do
 -- have been added.
 incrementalPush :: State -> Remote -> M.Map Ref Sha -> M.Map Ref Sha -> Annex (Bool, State)
 incrementalPush st rmt oldtrackingrefs newtrackingrefs = guardPush st $ do
-	bs <- calc [] (M.toList newtrackingrefs)
 	oldmanifest <- maybe (downloadManifestWhenPresent rmt) pure (manifestCache st)
-	bundlekey <- generateAndUploadGitBundle rmt bs oldmanifest
-	uploadManifest rmt (oldmanifest <> mkManifest [bundlekey] [])
-	return (True, st { manifestCache = Nothing })
+	if length (inManifest oldmanifest) + 1 > remoteAnnexMaxGitBundles (Remote.gitconfig rmt)
+		then fullPush' oldmanifest st rmt (M.keys newtrackingrefs)
+		else go oldmanifest
   where
+ 	go oldmanifest = do
+		bs <- calc [] (M.toList newtrackingrefs)
+		bundlekey <- generateAndUploadGitBundle rmt bs oldmanifest
+		uploadManifest rmt (oldmanifest <> mkManifest [bundlekey] [])
+		return (True, st { manifestCache = Nothing })
+	
 	calc c [] = return (reverse c)
 	calc c ((ref, sha):refs) = case M.lookup ref oldtrackingrefs of
 		Just oldsha

Types/GitConfig.hs

@@ -362,7 +362,6 @@ data RemoteGitConfig = RemoteGitConfig
 	, remoteAnnexStopCommand :: Maybe String
 	, remoteAnnexSpeculatePresent :: Bool
 	, remoteAnnexBare :: Maybe Bool
-	, remoteAnnexAllowEncryptedGitRepo :: Bool
 	, remoteAnnexRetry :: Maybe Integer
 	, remoteAnnexForwardRetry :: Maybe Integer
 	, remoteAnnexRetryDelay :: Maybe Seconds
@@ -374,6 +373,8 @@ data RemoteGitConfig = RemoteGitConfig
 	, remoteAnnexBwLimitDownload :: Maybe BwRate
 	, remoteAnnexAllowUnverifiedDownloads :: Bool
 	, remoteAnnexConfigUUID :: Maybe UUID
+	, remoteAnnexMaxGitBundles :: Int
+	, remoteAnnexAllowEncryptedGitRepo :: Bool
 
 	{- These settings are specific to particular types of remotes
 	 - including special remotes. -}
@@ -434,8 +435,6 @@ extractRemoteGitConfig r remotename = do
 		, remoteAnnexSpeculatePresent = 
 			getbool "speculate-present" False
 		, remoteAnnexBare = getmaybebool "bare"
-		, remoteAnnexAllowEncryptedGitRepo = 
-			getbool "allow-encrypted-gitrepo" False
 		, remoteAnnexRetry = getmayberead "retry"
 		, remoteAnnexForwardRetry = getmayberead "forward-retry"
 		, remoteAnnexRetryDelay = Seconds
@@ -480,6 +479,10 @@ extractRemoteGitConfig r remotename = do
 		, remoteAnnexDdarRepo = getmaybe "ddarrepo"
 		, remoteAnnexHookType = notempty $ getmaybe "hooktype"
 		, remoteAnnexExternalType = notempty $ getmaybe "externaltype"
+		, remoteAnnexMaxGitBundles = 
+			fromMaybe 100 (getmayberead  "max-git-bundles")
+		, remoteAnnexAllowEncryptedGitRepo = 
+			getbool "allow-encrypted-gitrepo" False
 		}
   where
 	getbool k d = fromMaybe d $ getmaybebool k

doc/git-annex.mdwn

@@ -1648,16 +1648,16 @@ Remotes are configured using these settings in `.git/config`.
   remotes, and is set when using [[git-annex-initremote]](1) with the
   `--private` option.
 
-* `remote.<name>.max-bundles`, `annex.max-bundles`
+* `remote.<name>.max-git-bundles`, `annex.max-git-bundles`
 
   When using [[git-remote-annex]] to store a git repository in a special 
   remote, this configures how many separate git bundle objects to store
-  in the special remote before re-pushing a single git bundle that contains
+  in the special remote before re-uploading a single git bundle that contains
   the entire git repository.
 
   The default is 100, which aims to avoid often needing to often re-upload,
   while preventing a new clone needing to download too many objects. Set to
-  0 to disable re-pushing.
+  0 to disable re-uploading.
 
 * `remote.<name>.allow-encrypted-gitrepo`
 

doc/git-remote-annex.mdwn

@@ -20,12 +20,6 @@ For example, to clone from a directory special remote:
 
     git clone annex::358ff77e-0bc3-11ef-bc49-872e6695c0e3?type=directory&encryption=none&directory=/mnt/foo/
 
-When a special remote needs some additional credentials to be provided,
-they are not included in the URL, and need to be provided when cloning from
-the special remote. That is typically done by setting environment
-variables. Some special remotes may also need environment variables to be
-set when pulling or pushing.
-
 When configuring the url of an existing special remote, a
 shorter url of "annex::" is sufficient. For example:
 
@@ -37,6 +31,12 @@ Configuring the url like that is automatically done when cloning from a
 special remote, but not by [[git-annex-initremote]](1) and
 [[git-annex-enableremote]](1).
 
+When a special remote needs some additional credentials to be provided,
+they are not included in the URL, and need to be provided when cloning from
+the special remote. That is typically done by setting environment
+variables. Some special remotes may also need environment variables to be
+set when pulling or pushing.
+
 The git repository is stored in the special remote using special annex objects
 with names starting with "GITMANIFEST--" and "GITBUNDLE--". For details about
 how the git repository is stored, see
@@ -46,7 +46,7 @@ Pushes to a special remote are usually done incrementally. However,
 sometimes the whole git repository (but not the annex) needs to be
 re-uploaded. That is done when deleting a ref from the remote. It's also
 done when too many git bundles accumulate in the special remote, as
-configured by the `remote.<name>.max-bundles` git config.
+configured by the `remote.<name>.max-git-bundles` git config.
 
 Like any git repository, a git repository stored on a special remote can
 have conflicting things pushed to it from different places. This mostly

doc/todo/git-remote-annex.mdwn

@@ -19,8 +19,6 @@ This is implememented and working. Remaining todo list for it:
 
 * Test incremental pushes that don't fast-forward.
 
-* Support max-bundles config
-
 * Cloning from an annex:: url with importtree=yes doesn't work
   (with or without exporttree=yes). This is because the ContentIdentifier
   db is not populated.