split upgrade into v9 and v10

v10 will run 1 year after the upgrade to v9, to give time for any v8
processes to die. Until that point, the v10 upgrade will be tried by
every process but deferred, so added support for deferring upgrades.

The upgrade prevention lock file that will be used by v10 is not yet
implemented, so it does not yet defer.

Sponsored-by: Dartmouth College's Datalad project

Annex/Version.hs

@@ -19,19 +19,19 @@ import qualified Annex
 import qualified Data.Map as M
 
 defaultVersion :: RepoVersion
-defaultVersion = RepoVersion 8
+defaultVersion = RepoVersion 10
 
 latestVersion :: RepoVersion
-latestVersion = RepoVersion 9
+latestVersion = RepoVersion 10
 
 supportedVersions :: [RepoVersion]
-supportedVersions = map RepoVersion [8, 9]
+supportedVersions = map RepoVersion [8, 9, 10]
 
 upgradeableVersions :: [RepoVersion]
 #ifndef mingw32_HOST_OS
-upgradeableVersions = map RepoVersion [0..8]
+upgradeableVersions = map RepoVersion [0..10]
 #else
-upgradeableVersions = map RepoVersion [2..8]
+upgradeableVersions = map RepoVersion [2..10]
 #endif
 
 autoUpgradeableVersions :: M.Map RepoVersion RepoVersion
@@ -41,6 +41,8 @@ autoUpgradeableVersions = M.fromList
 	, (RepoVersion 5, latestVersion)
 	, (RepoVersion 6, latestVersion)
 	, (RepoVersion 7, latestVersion)
+	, (RepoVersion 8, latestVersion) 
+	, (RepoVersion 9, latestVersion) 
 	]
 
 versionField :: ConfigKey
@@ -57,5 +59,5 @@ removeVersion = unsetConfig versionField
 
 versionNeedsWritableContentFiles :: Maybe RepoVersion -> Bool
 versionNeedsWritableContentFiles (Just v) 
-	| v >= RepoVersion 9 = False
+	| v >= RepoVersion 10 = False
 versionNeedsWritableContentFiles _ = True

Types/Upgrade.hs

@@ -0,0 +1,10 @@
+{- git-annex upgrade types
+ -
+ - Copyright 2022 Joey Hess <id@joeyh.name>
+ -
+ - Licensed under the GNU AGPL version 3 or higher.
+ -}
+
+module Types.Upgrade where
+
+data UpgradeResult = UpgradeSuccess | UpgradeFailed | UpgradeDeferred

Upgrade.hs

@@ -1,6 +1,6 @@
 {- git-annex upgrade support
  -
- - Copyright 2010-2020 Joey Hess <id@joeyh.name>
+ - Copyright 2010-2022 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
@@ -10,6 +10,7 @@
 module Upgrade where
 
 import Annex.Common
+import Types.Upgrade
 import qualified Annex
 import qualified Git
 import Config
@@ -27,6 +28,7 @@ import qualified Upgrade.V5
 import qualified Upgrade.V6
 import qualified Upgrade.V7
 import qualified Upgrade.V8
+import qualified Upgrade.V9
 
 import qualified Data.Map as M
 
@@ -63,25 +65,27 @@ needsUpgrade v
 
 upgrade :: Bool -> RepoVersion -> Annex Bool
 upgrade automatic destversion = do
-	upgraded <- go =<< getVersion
+	(upgraded, newversion) <- go =<< getVersion
-	when upgraded
+	when upgraded $
-		postupgrade
+		postupgrade newversion
 	return upgraded
   where
 	go (Just v)
-		| v >= destversion = return True
+		| v >= destversion = return (True, Just v)
 		| otherwise = ifM upgradingRemote
 			( upgraderemote
-			, ifM (up v)
+			, up v >>= \case
-				( go (Just (RepoVersion (fromRepoVersion v + 1)))
+				UpgradeSuccess -> go (Just (incrversion v) )
-				, return False
+				UpgradeFailed -> return (False, Just v)
-				)
+				UpgradeDeferred -> return (True, Just v)
 			)
-	go _ = return True
+	go Nothing = return (True, Nothing)
 
-	postupgrade = ifM upgradingRemote
+	incrversion v = RepoVersion (fromRepoVersion v + 1)
+
+	postupgrade newversion = ifM upgradingRemote
 		( reloadConfig
-		, setVersion destversion
+		, maybe noop setVersion newversion
 		)
 
 #ifndef mingw32_HOST_OS
@@ -98,7 +102,8 @@ upgrade automatic destversion = do
 	up (RepoVersion 6) = Upgrade.V6.upgrade automatic
 	up (RepoVersion 7) = Upgrade.V7.upgrade automatic
 	up (RepoVersion 8) = Upgrade.V8.upgrade automatic
-	up _ = return True
+	up (RepoVersion 9) = Upgrade.V9.upgrade automatic
+	up _ = return UpgradeDeferred
 
 	-- Upgrade local remotes by running git-annex upgrade in them.
 	-- This avoids complicating the upgrade code by needing to handle
@@ -111,8 +116,8 @@ upgrade automatic destversion = do
 			]
 			(\p -> p { cwd = Just rp })
 			(\_ _ _ pid -> waitForProcess pid >>= return . \case
-				ExitSuccess -> True
+				ExitSuccess -> (True, Nothing)
-				_ -> False
+				_ -> (False, Nothing)
 			)
 
 upgradingRemote :: Annex Bool

Upgrade/V0.hs

@@ -8,10 +8,11 @@
 module Upgrade.V0 where
 
 import Annex.Common
+import Types.Upgrade
 import Annex.Content
 import qualified Upgrade.V1
 
-upgrade :: Annex Bool
+upgrade :: Annex UpgradeResult
 upgrade = do
 	showAction "v0 to v1"
 

Upgrade/V1.hs

@@ -17,6 +17,7 @@ import qualified Data.ByteString.Lazy as L
 import qualified System.FilePath.ByteString as P
 
 import Annex.Common
+import Types.Upgrade
 import Annex.Content
 import Annex.Link
 import Annex.Perms
@@ -53,7 +54,7 @@ import qualified Upgrade.V2
 -- Something similar to the migrate subcommand could be used,
 -- and users could then run that at their leisure.
 
-upgrade :: Annex Bool
+upgrade :: Annex UpgradeResult
 upgrade = do
 	showAction "v1 to v2"
 	

Upgrade/V2.hs

@@ -8,6 +8,7 @@
 module Upgrade.V2 where
 
 import Annex.Common
+import Types.Upgrade
 import qualified Git
 import qualified Git.Command
 import qualified Git.Ref
@@ -38,7 +39,7 @@ olddir g
  - * Remove stuff that used to be needed in .gitattributes.
  - * Commit changes.
  -}
-upgrade :: Annex Bool
+upgrade :: Annex UpgradeResult
 upgrade = do
 	showAction "v2 to v3"
 	bare <- fromRepo Git.repoIsLocalBare
@@ -63,7 +64,7 @@ upgrade = do
 
 	unless bare push
 
-	return True
+	return UpgradeSuccess
 
 locationLogs :: Annex [(Key, FilePath)]
 locationLogs = do

Upgrade/V4.hs

@@ -8,8 +8,9 @@
 module Upgrade.V4 where
 
 import Annex.Common
+import Types.Upgrade
 
 {- Was only used for direct mode upgrade. v4 to v5 indirect update is a no-op,
  - and direct mode is no longer supported, so nothing needs to be done. -}
-upgrade :: Bool -> Annex Bool
+upgrade :: Bool -> Annex UpgradeResult
-upgrade _automatic = return True
+upgrade _automatic = return UpgradeSuccess

Upgrade/V5.hs

@@ -10,6 +10,7 @@
 module Upgrade.V5 where
 
 import Annex.Common
+import Types.Upgrade
 import Config
 import Config.Smudge
 import Annex.InodeSentinal
@@ -36,7 +37,7 @@ import qualified Utility.RawFilePath as R
 
 import qualified Data.ByteString as S
 
-upgrade :: Bool -> Annex Bool
+upgrade :: Bool -> Annex UpgradeResult
 upgrade automatic = flip catchNonAsync onexception $ do
 	unless automatic $
 		showAction "v5 to v6"
@@ -55,11 +56,11 @@ upgrade automatic = flip catchNonAsync onexception $ do
 	-- use direct mode may not have created it.
 	unlessM isDirect $
 		createInodeSentinalFile True
-	return True
+	return UpgradeSuccess
   where
 	onexception e = do
 		warning $ "caught exception: " ++ show e
-		return False
+		return UpgradeFailed
 
 -- git before 2.22 would OOM running git status on a large file.
 --

Upgrade/V6.hs

@@ -8,14 +8,15 @@
 module Upgrade.V6 where
 
 import Annex.Common
+import Types.Upgrade
 import Config
 import Annex.Hook
 
-upgrade :: Bool -> Annex Bool
+upgrade :: Bool -> Annex UpgradeResult
 upgrade automatic = do
 	unless automatic $
 		showAction "v6 to v7"
 	unlessM isBareRepo $ do
 		hookWrite postCheckoutHook
 		hookWrite postMergeHook
-	return True
+	return UpgradeSuccess

Upgrade/V7.hs

@@ -12,6 +12,7 @@ module Upgrade.V7 where
 
 import qualified Annex
 import Annex.Common
+import Types.Upgrade
 import Annex.CatFile
 import qualified Database.Keys
 import qualified Database.Keys.SQL
@@ -23,7 +24,7 @@ import qualified Utility.RawFilePath as R
 
 import qualified System.FilePath.ByteString as P
 
-upgrade :: Bool -> Annex Bool
+upgrade :: Bool -> Annex UpgradeResult
 upgrade automatic = do
 	unless automatic $
 		showAction "v7 to v8"
@@ -54,7 +55,7 @@ upgrade automatic = do
 	
 	updateSmudgeFilter
 
-	return True
+	return UpgradeSuccess
 
 gitAnnexKeysDbOld :: Git.Repo -> RawFilePath
 gitAnnexKeysDbOld r = gitAnnexDir r P.</> "keys"

Upgrade/V8.hs

@@ -8,37 +8,25 @@
 module Upgrade.V8 where
 
 import Annex.Common
-import Annex.Content
+import Types.Upgrade
-import Annex.Perms
+import Utility.Daemon
-import Git.ConfigTypes
-import Types.RepoVersion
 
-upgrade :: Bool -> Annex Bool
+upgrade :: Bool -> Annex UpgradeResult
 upgrade automatic = do
-	unless automatic $
+	-- Skip running when git-annex assistant (or watch) is running,
-		showAction "v8 to v9"
+	-- because these are long-running daemons that could conceivably
+	-- run for an entire year, and so still be running when the v10
+	-- upgrade happens. If the assistant then tried to drop a file
+	-- after the v10 upgrade, it would use the wrong content file
+	-- locking, which could lead to data loss. The remotedaemon does
+	-- not drop content, so will not block the upgrade.
+	pidfile <- fromRepo gitAnnexPidFile
+	liftIO (checkDaemon (fromRawFilePath pidfile)) >>= \case
+		Just _pid
+			| automatic -> return UpgradeDeferred
+			| otherwise -> giveup "Cannot upgrade to v9 when git-annex assistant or watch daemon is running."
+		Nothing -> do
+			unless automatic $
+				showAction "v8 to v9"
 
-	{- When core.sharedRepository is set, object files
+			return UpgradeSuccess
-	 - used to have their write bits set. That can now be removed,
-	 - if the user the upgrade is running as has permission to remove
-	 - it. (Otherwise, a later fsck will fix up the permissions.) -}
-	withShared $ \sr -> case sr of
-		GroupShared -> removewrite sr
-		AllShared -> removewrite sr
-		_ -> return ()
-
-	return True
-  where
-	newver = Just (RepoVersion 9)
-
-	removewrite sr = do
-		ks <- listKeys InAnnex
-		forM_ ks $ \k -> do
-			obj <- calcRepo (gitAnnexLocation k)
-			keystatus <- getKeyStatus k
-			case keystatus of
-				KeyPresent -> void $ tryIO $
-					freezeContent'' sr obj newver
-				KeyUnlockedThin -> return ()
-				KeyLockedThin -> return ()
-				KeyMissing -> return ()

Upgrade/V9.hs

@@ -0,0 +1,45 @@
+{- git-annex v9 -> v10 upgrade support
+ -
+ - Copyright 2022 Joey Hess <id@joeyh.name>
+ -
+ - Licensed under the GNU AGPL version 3 or higher.
+ -}
+
+module Upgrade.V9 where
+
+import Annex.Common
+import Types.Upgrade
+import Annex.Content
+import Annex.Perms
+import Git.ConfigTypes
+import Types.RepoVersion
+
+upgrade :: Bool -> Annex UpgradeResult
+upgrade automatic = do
+	unless automatic $
+		showAction "v9 to v10"
+
+	{- When core.sharedRepository is set, object files
+	 - used to have their write bits set. That can now be removed,
+	 - if the user the upgrade is running as has permission to remove
+	 - it. (Otherwise, a later fsck will fix up the permissions.) -}
+	withShared $ \sr -> case sr of
+		GroupShared -> removewrite sr
+		AllShared -> removewrite sr
+		_ -> return ()
+
+	return UpgradeDeferred
+  where
+	newver = Just (RepoVersion 9)
+
+	removewrite sr = do
+		ks <- listKeys InAnnex
+		forM_ ks $ \k -> do
+			obj <- calcRepo (gitAnnexLocation k)
+			keystatus <- getKeyStatus k
+			case keystatus of
+				KeyPresent -> void $ tryIO $
+					freezeContent'' sr obj newver
+				KeyUnlockedThin -> return ()
+				KeyLockedThin -> return ()
+				KeyMissing -> return ()

doc/upgrades.mdwn

@@ -49,6 +49,24 @@ the upgrade would need to be run in a copy of the repository.
 
 The upgrade events, so far:
 
+## v9 -> v10 (git-annex version 10.x)
+
+v10 repositories change a fundamental part of how git-annex does locking.
+
+v9 repositories are automatically upgraded to v10, but with a delay.
+The upgrade happens one year after the repository was upgraded to v9.
+This delay is because it would not be safe to upgrade to v10 if a
+git-annex process version 8.x was still running. Waiting a year makes
+it very unlikely that such a process is still running.
+
+## v8 -> v9 (git-annex version 10.x)
+
+v8 repositories are automatically upgraded to v9.
+
+Starting in v9, upgrades to a repository are prevented while another
+git-annex command is running. That is needed for the v10 repository
+upgrade to be performed safely.
+
 ## v7 -> v8 (git-annex version 8.x)
 
 v7 repositories are automatically upgraded to v8.

git-annex.cabal

@@ -1055,6 +1055,7 @@ Executable git-annex
     Types.Transitions
     Types.TrustLevel
     Types.UUID
+    Types.Upgrade
     Types.UrlContents
     Types.VectorClock
     Types.View
@@ -1070,6 +1071,7 @@ Executable git-annex
     Upgrade.V6
     Upgrade.V7
     Upgrade.V8
+    Upgrade.V9
     Utility.Aeson
     Utility.Android
     Utility.Applicative