From dc9295017f760fef55cbf7001a7744338549600b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 6 Nov 2019 15:37:18 -0400 Subject: [PATCH] v8 upgrade of keys db Renamed the database to .git/annex/keysdb; the old .git/annex/keys gets deleted during the upgrade. It is possible that an old git-annex process is running during the upgrade. If so, it will be able to continue using the old keys db until the upgrade is complete, and then will presumably fail in some ugly way. Or perhaps the upgrade will be unable to delete the open files on some systems, and so fail with an ugly error message. It's also possible for multiple processes to be running the upgrade concurrently. That should be fine; they will both write the same information into the keys db. Other databases still need to be upgraded. --- Annex/Locations.hs | 6 ++-- Annex/Version.hs | 7 ++-- Git/LsFiles.hs | 1 + Upgrade.hs | 2 ++ Upgrade/V7.hs | 89 ++++++++++++++++++++++++++++++++++++++++++++++ git-annex.cabal | 1 + 6 files changed, 100 insertions(+), 6 deletions(-) create mode 100644 Upgrade/V7.hs diff --git a/Annex/Locations.hs b/Annex/Locations.hs index 96ab104c54..4020e000c9 100644 --- a/Annex/Locations.hs +++ b/Annex/Locations.hs @@ -304,16 +304,16 @@ gitAnnexBadLocation key r = gitAnnexBadDir r keyFile key gitAnnexUnusedLog :: FilePath -> Git.Repo -> FilePath gitAnnexUnusedLog prefix r = gitAnnexDir r (prefix ++ "unused") -{- .git/annex/keys/ contains a database of information about keys. -} +{- .git/annex/keysdb/ contains a database of information about keys. -} gitAnnexKeysDb :: Git.Repo -> FilePath -gitAnnexKeysDb r = gitAnnexDir r "keys" +gitAnnexKeysDb r = gitAnnexDir r "keysdb" {- Lock file for the keys database. -} gitAnnexKeysDbLock :: Git.Repo -> FilePath gitAnnexKeysDbLock r = gitAnnexKeysDb r ++ ".lck" {- Contains the stat of the last index file that was - - reconciled with rhe keys database. -} + - reconciled with the keys database. -} gitAnnexKeysDbIndexCache :: Git.Repo -> FilePath gitAnnexKeysDbIndexCache r = gitAnnexKeysDb r ++ ".cache" diff --git a/Annex/Version.hs b/Annex/Version.hs index 27bc95db19..4a8e2f9840 100644 --- a/Annex/Version.hs +++ b/Annex/Version.hs @@ -1,6 +1,6 @@ {- git-annex repository versioning - - - Copyright 2010-2018 Joey Hess + - Copyright 2010-2019 Joey Hess - - Licensed under the GNU AGPL version 3 or higher. -} @@ -27,9 +27,9 @@ supportedVersions = map RepoVersion [8] upgradableVersions :: [RepoVersion] #ifndef mingw32_HOST_OS -upgradableVersions = map RepoVersion [0..6] +upgradableVersions = map RepoVersion [0..7] #else -upgradableVersions = map RepoVersion [2..6] +upgradableVersions = map RepoVersion [2..7] #endif autoUpgradeableVersions :: M.Map RepoVersion RepoVersion @@ -38,6 +38,7 @@ autoUpgradeableVersions = M.fromList , (RepoVersion 4, RepoVersion 5) , (RepoVersion 5, RepoVersion 6) , (RepoVersion 6, RepoVersion 7) + -- , (RepoVersion 7, RepoVersion 8) ] versionField :: ConfigKey diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs index 17527be138..f5f74a22a5 100644 --- a/Git/LsFiles.hs +++ b/Git/LsFiles.hs @@ -24,6 +24,7 @@ module Git.LsFiles ( Unmerged(..), unmerged, StagedDetails, + inodeCaches, ) where import Common diff --git a/Upgrade.hs b/Upgrade.hs index 1cde059521..7f54e8b334 100644 --- a/Upgrade.hs +++ b/Upgrade.hs @@ -23,6 +23,7 @@ import qualified Upgrade.V3 import qualified Upgrade.V4 import qualified Upgrade.V5 import qualified Upgrade.V6 +import qualified Upgrade.V7 import qualified Data.Map as M @@ -84,5 +85,6 @@ upgrade automatic destversion = do up (RepoVersion 4) = Upgrade.V4.upgrade automatic up (RepoVersion 5) = Upgrade.V5.upgrade automatic up (RepoVersion 6) = Upgrade.V6.upgrade automatic + up (RepoVersion 7) = Upgrade.V7.upgrade automatic up _ = return True diff --git a/Upgrade/V7.hs b/Upgrade/V7.hs new file mode 100644 index 0000000000..fcaca64e8a --- /dev/null +++ b/Upgrade/V7.hs @@ -0,0 +1,89 @@ +{- git-annex v7 -> v8 upgrade support + - + - Copyright 2019 Joey Hess + - + - Licensed under the GNU AGPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} + +module Upgrade.V7 where + +import Annex.Common +import Annex.CatFile +import qualified Database.Keys +import qualified Database.Keys.SQL +import qualified Git.LsFiles as LsFiles +import qualified Git +import Git.FilePath + +upgrade :: Bool -> Annex Bool +upgrade automatic = do + unless automatic $ + showAction "v7 to v8" + + populateKeysDb + + removeOldDb gitAnnexKeysDbOld + liftIO . nukeFile =<< fromRepo gitAnnexKeysDbIndexCacheOld + liftIO . nukeFile =<< fromRepo gitAnnexKeysDbLockOld + + return True + +gitAnnexKeysDbOld :: Git.Repo -> FilePath +gitAnnexKeysDbOld r = gitAnnexDir r "keys" + +gitAnnexKeysDbLockOld :: Git.Repo -> FilePath +gitAnnexKeysDbLockOld r = gitAnnexKeysDbOld r ++ ".lck" + +gitAnnexKeysDbIndexCacheOld :: Git.Repo -> FilePath +gitAnnexKeysDbIndexCacheOld r = gitAnnexKeysDbOld r ++ ".cache" + +removeOldDb :: (Git.Repo -> FilePath) -> Annex () +removeOldDb getdb = do + db <- fromRepo getdb + whenM (liftIO $ doesDirectoryExist db) $ do + v <- liftIO $ tryNonAsync $ +#if MIN_VERSION_directory(1,2,7) + removePathForcibly db +#else + removeDirectoryRecursive db +#endif + case v of + Left ex -> giveup $ "Failed removing old database directory " ++ db ++ " during upgrade (" ++ show ex ++ ") -- delete that and re-run git-annex to finish the upgrade." + Right () -> return () + +-- Populate the new keys database with associated files and inode caches. +-- +-- The information is queried from git. The index contains inode cache +-- information for all staged files, so that is used. +-- +-- Note that typically the inode cache of annex objects is also stored in +-- the keys database. This does not add it though, because it's possible +-- that any annex object has gotten modified. The most likely way would be +-- due to annex.thin having been set at some point in the past, bypassing +-- the usual safeguards against object modification. When a worktree file +-- is still a hardlink to an annex object, then they have the same inode +-- cache, so using the inode cache from the git index will get the right +-- thing added in that case. But there are cases where the annex object's +-- inode cache is not added here, most notably when it's not unlocked. +-- The result will be more work needing to be done by isUnmodified and +-- by inAnnex (the latter only when annex.thin is set) to verify the +-- annex object. That work is only done once, and then the object will +-- finally get its inode cached. +populateKeysDb :: Annex () +populateKeysDb = do + top <- fromRepo Git.repoPath + (l, cleanup) <- inRepo $ LsFiles.inodeCaches [top] + forM_ l $ \case + (_f, Nothing) -> giveup "Unable to parse git ls-files --debug output while upgrading git-annex sqlite databases." + (f, Just ic) -> unlessM (liftIO $ isSymbolicLink <$> getSymbolicLinkStatus f) $ do + catKeyFile f >>= \case + Nothing -> noop + Just k -> do + topf <- inRepo $ toTopFilePath f + Database.Keys.runWriter $ \h -> liftIO $ do + Database.Keys.SQL.addAssociatedFileFast k topf h + Database.Keys.SQL.addInodeCaches k [ic] h + liftIO $ void cleanup + Database.Keys.closeDb diff --git a/git-annex.cabal b/git-annex.cabal index 1f18d6dca1..10f294df7d 100644 --- a/git-annex.cabal +++ b/git-annex.cabal @@ -1017,6 +1017,7 @@ Executable git-annex Upgrade.V5 Upgrade.V5.Direct Upgrade.V6 + Upgrade.V7 Utility.Aeson Utility.Android Utility.Applicative