From 856ce5cf5f4e6102cff10214963b4da1618c1c65 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 19 Jan 2022 13:06:31 -0400 Subject: [PATCH] split upgrade into v9 and v10 v10 will run 1 year after the upgrade to v9, to give time for any v8 processes to die. Until that point, the v10 upgrade will be tried by every process but deferred, so added support for deferring upgrades. The upgrade prevention lock file that will be used by v10 is not yet implemented, so it does not yet defer. Sponsored-by: Dartmouth College's Datalad project --- Annex/Version.hs | 14 +++++++------ Types/Upgrade.hs | 10 ++++++++++ Upgrade.hs | 35 +++++++++++++++++++-------------- Upgrade/V0.hs | 3 ++- Upgrade/V1.hs | 3 ++- Upgrade/V2.hs | 5 +++-- Upgrade/V4.hs | 5 +++-- Upgrade/V5.hs | 7 ++++--- Upgrade/V6.hs | 5 +++-- Upgrade/V7.hs | 5 +++-- Upgrade/V8.hs | 50 ++++++++++++++++++----------------------------- Upgrade/V9.hs | 45 ++++++++++++++++++++++++++++++++++++++++++ doc/upgrades.mdwn | 18 +++++++++++++++++ git-annex.cabal | 2 ++ 14 files changed, 142 insertions(+), 65 deletions(-) create mode 100644 Types/Upgrade.hs create mode 100644 Upgrade/V9.hs diff --git a/Annex/Version.hs b/Annex/Version.hs index 27a4cb832b..c8c47f84f2 100644 --- a/Annex/Version.hs +++ b/Annex/Version.hs @@ -19,19 +19,19 @@ import qualified Annex import qualified Data.Map as M defaultVersion :: RepoVersion -defaultVersion = RepoVersion 8 +defaultVersion = RepoVersion 10 latestVersion :: RepoVersion -latestVersion = RepoVersion 9 +latestVersion = RepoVersion 10 supportedVersions :: [RepoVersion] -supportedVersions = map RepoVersion [8, 9] +supportedVersions = map RepoVersion [8, 9, 10] upgradeableVersions :: [RepoVersion] #ifndef mingw32_HOST_OS -upgradeableVersions = map RepoVersion [0..8] +upgradeableVersions = map RepoVersion [0..10] #else -upgradeableVersions = map RepoVersion [2..8] +upgradeableVersions = map RepoVersion [2..10] #endif autoUpgradeableVersions :: M.Map RepoVersion RepoVersion @@ -41,6 +41,8 @@ autoUpgradeableVersions = M.fromList , (RepoVersion 5, latestVersion) , (RepoVersion 6, latestVersion) , (RepoVersion 7, latestVersion) + , (RepoVersion 8, latestVersion) + , (RepoVersion 9, latestVersion) ] versionField :: ConfigKey @@ -57,5 +59,5 @@ removeVersion = unsetConfig versionField versionNeedsWritableContentFiles :: Maybe RepoVersion -> Bool versionNeedsWritableContentFiles (Just v) - | v >= RepoVersion 9 = False + | v >= RepoVersion 10 = False versionNeedsWritableContentFiles _ = True diff --git a/Types/Upgrade.hs b/Types/Upgrade.hs new file mode 100644 index 0000000000..0249437443 --- /dev/null +++ b/Types/Upgrade.hs @@ -0,0 +1,10 @@ +{- git-annex upgrade types + - + - Copyright 2022 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +module Types.Upgrade where + +data UpgradeResult = UpgradeSuccess | UpgradeFailed | UpgradeDeferred diff --git a/Upgrade.hs b/Upgrade.hs index d87a7b0302..c8a7c7a748 100644 --- a/Upgrade.hs +++ b/Upgrade.hs @@ -1,6 +1,6 @@ {- git-annex upgrade support - - - Copyright 2010-2020 Joey Hess + - Copyright 2010-2022 Joey Hess - - Licensed under the GNU AGPL version 3 or higher. -} @@ -10,6 +10,7 @@ module Upgrade where import Annex.Common +import Types.Upgrade import qualified Annex import qualified Git import Config @@ -27,6 +28,7 @@ import qualified Upgrade.V5 import qualified Upgrade.V6 import qualified Upgrade.V7 import qualified Upgrade.V8 +import qualified Upgrade.V9 import qualified Data.Map as M @@ -63,25 +65,27 @@ needsUpgrade v upgrade :: Bool -> RepoVersion -> Annex Bool upgrade automatic destversion = do - upgraded <- go =<< getVersion - when upgraded - postupgrade + (upgraded, newversion) <- go =<< getVersion + when upgraded $ + postupgrade newversion return upgraded where go (Just v) - | v >= destversion = return True + | v >= destversion = return (True, Just v) | otherwise = ifM upgradingRemote ( upgraderemote - , ifM (up v) - ( go (Just (RepoVersion (fromRepoVersion v + 1))) - , return False - ) + , up v >>= \case + UpgradeSuccess -> go (Just (incrversion v) ) + UpgradeFailed -> return (False, Just v) + UpgradeDeferred -> return (True, Just v) ) - go _ = return True + go Nothing = return (True, Nothing) - postupgrade = ifM upgradingRemote + incrversion v = RepoVersion (fromRepoVersion v + 1) + + postupgrade newversion = ifM upgradingRemote ( reloadConfig - , setVersion destversion + , maybe noop setVersion newversion ) #ifndef mingw32_HOST_OS @@ -98,7 +102,8 @@ upgrade automatic destversion = do up (RepoVersion 6) = Upgrade.V6.upgrade automatic up (RepoVersion 7) = Upgrade.V7.upgrade automatic up (RepoVersion 8) = Upgrade.V8.upgrade automatic - up _ = return True + up (RepoVersion 9) = Upgrade.V9.upgrade automatic + up _ = return UpgradeDeferred -- Upgrade local remotes by running git-annex upgrade in them. -- This avoids complicating the upgrade code by needing to handle @@ -111,8 +116,8 @@ upgrade automatic destversion = do ] (\p -> p { cwd = Just rp }) (\_ _ _ pid -> waitForProcess pid >>= return . \case - ExitSuccess -> True - _ -> False + ExitSuccess -> (True, Nothing) + _ -> (False, Nothing) ) upgradingRemote :: Annex Bool diff --git a/Upgrade/V0.hs b/Upgrade/V0.hs index ca8601e224..3e6d9e6202 100644 --- a/Upgrade/V0.hs +++ b/Upgrade/V0.hs @@ -8,10 +8,11 @@ module Upgrade.V0 where import Annex.Common +import Types.Upgrade import Annex.Content import qualified Upgrade.V1 -upgrade :: Annex Bool +upgrade :: Annex UpgradeResult upgrade = do showAction "v0 to v1" diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs index bb2abd743d..107db03615 100644 --- a/Upgrade/V1.hs +++ b/Upgrade/V1.hs @@ -17,6 +17,7 @@ import qualified Data.ByteString.Lazy as L import qualified System.FilePath.ByteString as P import Annex.Common +import Types.Upgrade import Annex.Content import Annex.Link import Annex.Perms @@ -53,7 +54,7 @@ import qualified Upgrade.V2 -- Something similar to the migrate subcommand could be used, -- and users could then run that at their leisure. -upgrade :: Annex Bool +upgrade :: Annex UpgradeResult upgrade = do showAction "v1 to v2" diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs index 6ba538634a..13bd191cb2 100644 --- a/Upgrade/V2.hs +++ b/Upgrade/V2.hs @@ -8,6 +8,7 @@ module Upgrade.V2 where import Annex.Common +import Types.Upgrade import qualified Git import qualified Git.Command import qualified Git.Ref @@ -38,7 +39,7 @@ olddir g - * Remove stuff that used to be needed in .gitattributes. - * Commit changes. -} -upgrade :: Annex Bool +upgrade :: Annex UpgradeResult upgrade = do showAction "v2 to v3" bare <- fromRepo Git.repoIsLocalBare @@ -63,7 +64,7 @@ upgrade = do unless bare push - return True + return UpgradeSuccess locationLogs :: Annex [(Key, FilePath)] locationLogs = do diff --git a/Upgrade/V4.hs b/Upgrade/V4.hs index bb3cca044d..feeadefe1e 100644 --- a/Upgrade/V4.hs +++ b/Upgrade/V4.hs @@ -8,8 +8,9 @@ module Upgrade.V4 where import Annex.Common +import Types.Upgrade {- Was only used for direct mode upgrade. v4 to v5 indirect update is a no-op, - and direct mode is no longer supported, so nothing needs to be done. -} -upgrade :: Bool -> Annex Bool -upgrade _automatic = return True +upgrade :: Bool -> Annex UpgradeResult +upgrade _automatic = return UpgradeSuccess diff --git a/Upgrade/V5.hs b/Upgrade/V5.hs index 2db92d57f9..f3b1de01cb 100644 --- a/Upgrade/V5.hs +++ b/Upgrade/V5.hs @@ -10,6 +10,7 @@ module Upgrade.V5 where import Annex.Common +import Types.Upgrade import Config import Config.Smudge import Annex.InodeSentinal @@ -36,7 +37,7 @@ import qualified Utility.RawFilePath as R import qualified Data.ByteString as S -upgrade :: Bool -> Annex Bool +upgrade :: Bool -> Annex UpgradeResult upgrade automatic = flip catchNonAsync onexception $ do unless automatic $ showAction "v5 to v6" @@ -55,11 +56,11 @@ upgrade automatic = flip catchNonAsync onexception $ do -- use direct mode may not have created it. unlessM isDirect $ createInodeSentinalFile True - return True + return UpgradeSuccess where onexception e = do warning $ "caught exception: " ++ show e - return False + return UpgradeFailed -- git before 2.22 would OOM running git status on a large file. -- diff --git a/Upgrade/V6.hs b/Upgrade/V6.hs index 95f5f0d45e..54a559ad85 100644 --- a/Upgrade/V6.hs +++ b/Upgrade/V6.hs @@ -8,14 +8,15 @@ module Upgrade.V6 where import Annex.Common +import Types.Upgrade import Config import Annex.Hook -upgrade :: Bool -> Annex Bool +upgrade :: Bool -> Annex UpgradeResult upgrade automatic = do unless automatic $ showAction "v6 to v7" unlessM isBareRepo $ do hookWrite postCheckoutHook hookWrite postMergeHook - return True + return UpgradeSuccess diff --git a/Upgrade/V7.hs b/Upgrade/V7.hs index b1fe9c639f..467d55442b 100644 --- a/Upgrade/V7.hs +++ b/Upgrade/V7.hs @@ -12,6 +12,7 @@ module Upgrade.V7 where import qualified Annex import Annex.Common +import Types.Upgrade import Annex.CatFile import qualified Database.Keys import qualified Database.Keys.SQL @@ -23,7 +24,7 @@ import qualified Utility.RawFilePath as R import qualified System.FilePath.ByteString as P -upgrade :: Bool -> Annex Bool +upgrade :: Bool -> Annex UpgradeResult upgrade automatic = do unless automatic $ showAction "v7 to v8" @@ -54,7 +55,7 @@ upgrade automatic = do updateSmudgeFilter - return True + return UpgradeSuccess gitAnnexKeysDbOld :: Git.Repo -> RawFilePath gitAnnexKeysDbOld r = gitAnnexDir r P. "keys" diff --git a/Upgrade/V8.hs b/Upgrade/V8.hs index 82dba93950..41cf315131 100644 --- a/Upgrade/V8.hs +++ b/Upgrade/V8.hs @@ -8,37 +8,25 @@ module Upgrade.V8 where import Annex.Common -import Annex.Content -import Annex.Perms -import Git.ConfigTypes -import Types.RepoVersion +import Types.Upgrade +import Utility.Daemon -upgrade :: Bool -> Annex Bool +upgrade :: Bool -> Annex UpgradeResult upgrade automatic = do - unless automatic $ - showAction "v8 to v9" + -- Skip running when git-annex assistant (or watch) is running, + -- because these are long-running daemons that could conceivably + -- run for an entire year, and so still be running when the v10 + -- upgrade happens. If the assistant then tried to drop a file + -- after the v10 upgrade, it would use the wrong content file + -- locking, which could lead to data loss. The remotedaemon does + -- not drop content, so will not block the upgrade. + pidfile <- fromRepo gitAnnexPidFile + liftIO (checkDaemon (fromRawFilePath pidfile)) >>= \case + Just _pid + | automatic -> return UpgradeDeferred + | otherwise -> giveup "Cannot upgrade to v9 when git-annex assistant or watch daemon is running." + Nothing -> do + unless automatic $ + showAction "v8 to v9" - {- When core.sharedRepository is set, object files - - used to have their write bits set. That can now be removed, - - if the user the upgrade is running as has permission to remove - - it. (Otherwise, a later fsck will fix up the permissions.) -} - withShared $ \sr -> case sr of - GroupShared -> removewrite sr - AllShared -> removewrite sr - _ -> return () - - return True - where - newver = Just (RepoVersion 9) - - removewrite sr = do - ks <- listKeys InAnnex - forM_ ks $ \k -> do - obj <- calcRepo (gitAnnexLocation k) - keystatus <- getKeyStatus k - case keystatus of - KeyPresent -> void $ tryIO $ - freezeContent'' sr obj newver - KeyUnlockedThin -> return () - KeyLockedThin -> return () - KeyMissing -> return () + return UpgradeSuccess diff --git a/Upgrade/V9.hs b/Upgrade/V9.hs new file mode 100644 index 0000000000..6208e5e0fd --- /dev/null +++ b/Upgrade/V9.hs @@ -0,0 +1,45 @@ +{- git-annex v9 -> v10 upgrade support + - + - Copyright 2022 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +module Upgrade.V9 where + +import Annex.Common +import Types.Upgrade +import Annex.Content +import Annex.Perms +import Git.ConfigTypes +import Types.RepoVersion + +upgrade :: Bool -> Annex UpgradeResult +upgrade automatic = do + unless automatic $ + showAction "v9 to v10" + + {- When core.sharedRepository is set, object files + - used to have their write bits set. That can now be removed, + - if the user the upgrade is running as has permission to remove + - it. (Otherwise, a later fsck will fix up the permissions.) -} + withShared $ \sr -> case sr of + GroupShared -> removewrite sr + AllShared -> removewrite sr + _ -> return () + + return UpgradeDeferred + where + newver = Just (RepoVersion 9) + + removewrite sr = do + ks <- listKeys InAnnex + forM_ ks $ \k -> do + obj <- calcRepo (gitAnnexLocation k) + keystatus <- getKeyStatus k + case keystatus of + KeyPresent -> void $ tryIO $ + freezeContent'' sr obj newver + KeyUnlockedThin -> return () + KeyLockedThin -> return () + KeyMissing -> return () diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn index 930e571a5a..794c7464bc 100644 --- a/doc/upgrades.mdwn +++ b/doc/upgrades.mdwn @@ -49,6 +49,24 @@ the upgrade would need to be run in a copy of the repository. The upgrade events, so far: +## v9 -> v10 (git-annex version 10.x) + +v10 repositories change a fundamental part of how git-annex does locking. + +v9 repositories are automatically upgraded to v10, but with a delay. +The upgrade happens one year after the repository was upgraded to v9. +This delay is because it would not be safe to upgrade to v10 if a +git-annex process version 8.x was still running. Waiting a year makes +it very unlikely that such a process is still running. + +## v8 -> v9 (git-annex version 10.x) + +v8 repositories are automatically upgraded to v9. + +Starting in v9, upgrades to a repository are prevented while another +git-annex command is running. That is needed for the v10 repository +upgrade to be performed safely. + ## v7 -> v8 (git-annex version 8.x) v7 repositories are automatically upgraded to v8. diff --git a/git-annex.cabal b/git-annex.cabal index 1a4f3707cf..9aa5b5033c 100644 --- a/git-annex.cabal +++ b/git-annex.cabal @@ -1055,6 +1055,7 @@ Executable git-annex Types.Transitions Types.TrustLevel Types.UUID + Types.Upgrade Types.UrlContents Types.VectorClock Types.View @@ -1070,6 +1071,7 @@ Executable git-annex Upgrade.V6 Upgrade.V7 Upgrade.V8 + Upgrade.V9 Utility.Aeson Utility.Android Utility.Applicative