v8 upgrade of keys db

Renamed the database to .git/annex/keysdb;
the old .git/annex/keys gets deleted during the upgrade.

It is possible that an old git-annex process is running during the
upgrade. If so, it will be able to continue using the old keys db until the
upgrade is complete, and then will presumably fail in some ugly way. Or
perhaps the upgrade will be unable to delete the open files on some
systems, and so fail with an ugly error message.

It's also possible for multiple processes to be running the upgrade
concurrently. That should be fine; they will both write the same
information into the keys db.

Other databases still need to be upgraded.
This commit is contained in:
Joey Hess 2019-11-06 15:37:18 -04:00
parent 09ee6b0ccb
commit dc9295017f
No known key found for this signature in database
GPG key ID: DB12DB0FF05F8F38
6 changed files with 100 additions and 6 deletions

View file

@ -304,16 +304,16 @@ gitAnnexBadLocation key r = gitAnnexBadDir r </> keyFile key
gitAnnexUnusedLog :: FilePath -> Git.Repo -> FilePath
gitAnnexUnusedLog prefix r = gitAnnexDir r </> (prefix ++ "unused")
{- .git/annex/keys/ contains a database of information about keys. -}
{- .git/annex/keysdb/ contains a database of information about keys. -}
gitAnnexKeysDb :: Git.Repo -> FilePath
gitAnnexKeysDb r = gitAnnexDir r </> "keys"
gitAnnexKeysDb r = gitAnnexDir r </> "keysdb"
{- Lock file for the keys database. -}
gitAnnexKeysDbLock :: Git.Repo -> FilePath
gitAnnexKeysDbLock r = gitAnnexKeysDb r ++ ".lck"
{- Contains the stat of the last index file that was
- reconciled with rhe keys database. -}
- reconciled with the keys database. -}
gitAnnexKeysDbIndexCache :: Git.Repo -> FilePath
gitAnnexKeysDbIndexCache r = gitAnnexKeysDb r ++ ".cache"

View file

@ -1,6 +1,6 @@
{- git-annex repository versioning
-
- Copyright 2010-2018 Joey Hess <id@joeyh.name>
- Copyright 2010-2019 Joey Hess <id@joeyh.name>
-
- Licensed under the GNU AGPL version 3 or higher.
-}
@ -27,9 +27,9 @@ supportedVersions = map RepoVersion [8]
upgradableVersions :: [RepoVersion]
#ifndef mingw32_HOST_OS
upgradableVersions = map RepoVersion [0..6]
upgradableVersions = map RepoVersion [0..7]
#else
upgradableVersions = map RepoVersion [2..6]
upgradableVersions = map RepoVersion [2..7]
#endif
autoUpgradeableVersions :: M.Map RepoVersion RepoVersion
@ -38,6 +38,7 @@ autoUpgradeableVersions = M.fromList
, (RepoVersion 4, RepoVersion 5)
, (RepoVersion 5, RepoVersion 6)
, (RepoVersion 6, RepoVersion 7)
-- , (RepoVersion 7, RepoVersion 8)
]
versionField :: ConfigKey

View file

@ -24,6 +24,7 @@ module Git.LsFiles (
Unmerged(..),
unmerged,
StagedDetails,
inodeCaches,
) where
import Common

View file

@ -23,6 +23,7 @@ import qualified Upgrade.V3
import qualified Upgrade.V4
import qualified Upgrade.V5
import qualified Upgrade.V6
import qualified Upgrade.V7
import qualified Data.Map as M
@ -84,5 +85,6 @@ upgrade automatic destversion = do
up (RepoVersion 4) = Upgrade.V4.upgrade automatic
up (RepoVersion 5) = Upgrade.V5.upgrade automatic
up (RepoVersion 6) = Upgrade.V6.upgrade automatic
up (RepoVersion 7) = Upgrade.V7.upgrade automatic
up _ = return True

89
Upgrade/V7.hs Normal file
View file

@ -0,0 +1,89 @@
{- git-annex v7 -> v8 upgrade support
-
- Copyright 2019 Joey Hess <id@joeyh.name>
-
- Licensed under the GNU AGPL version 3 or higher.
-}
{-# LANGUAGE CPP #-}
module Upgrade.V7 where
import Annex.Common
import Annex.CatFile
import qualified Database.Keys
import qualified Database.Keys.SQL
import qualified Git.LsFiles as LsFiles
import qualified Git
import Git.FilePath
upgrade :: Bool -> Annex Bool
upgrade automatic = do
unless automatic $
showAction "v7 to v8"
populateKeysDb
removeOldDb gitAnnexKeysDbOld
liftIO . nukeFile =<< fromRepo gitAnnexKeysDbIndexCacheOld
liftIO . nukeFile =<< fromRepo gitAnnexKeysDbLockOld
return True
gitAnnexKeysDbOld :: Git.Repo -> FilePath
gitAnnexKeysDbOld r = gitAnnexDir r </> "keys"
gitAnnexKeysDbLockOld :: Git.Repo -> FilePath
gitAnnexKeysDbLockOld r = gitAnnexKeysDbOld r ++ ".lck"
gitAnnexKeysDbIndexCacheOld :: Git.Repo -> FilePath
gitAnnexKeysDbIndexCacheOld r = gitAnnexKeysDbOld r ++ ".cache"
removeOldDb :: (Git.Repo -> FilePath) -> Annex ()
removeOldDb getdb = do
db <- fromRepo getdb
whenM (liftIO $ doesDirectoryExist db) $ do
v <- liftIO $ tryNonAsync $
#if MIN_VERSION_directory(1,2,7)
removePathForcibly db
#else
removeDirectoryRecursive db
#endif
case v of
Left ex -> giveup $ "Failed removing old database directory " ++ db ++ " during upgrade (" ++ show ex ++ ") -- delete that and re-run git-annex to finish the upgrade."
Right () -> return ()
-- Populate the new keys database with associated files and inode caches.
--
-- The information is queried from git. The index contains inode cache
-- information for all staged files, so that is used.
--
-- Note that typically the inode cache of annex objects is also stored in
-- the keys database. This does not add it though, because it's possible
-- that any annex object has gotten modified. The most likely way would be
-- due to annex.thin having been set at some point in the past, bypassing
-- the usual safeguards against object modification. When a worktree file
-- is still a hardlink to an annex object, then they have the same inode
-- cache, so using the inode cache from the git index will get the right
-- thing added in that case. But there are cases where the annex object's
-- inode cache is not added here, most notably when it's not unlocked.
-- The result will be more work needing to be done by isUnmodified and
-- by inAnnex (the latter only when annex.thin is set) to verify the
-- annex object. That work is only done once, and then the object will
-- finally get its inode cached.
populateKeysDb :: Annex ()
populateKeysDb = do
top <- fromRepo Git.repoPath
(l, cleanup) <- inRepo $ LsFiles.inodeCaches [top]
forM_ l $ \case
(_f, Nothing) -> giveup "Unable to parse git ls-files --debug output while upgrading git-annex sqlite databases."
(f, Just ic) -> unlessM (liftIO $ isSymbolicLink <$> getSymbolicLinkStatus f) $ do
catKeyFile f >>= \case
Nothing -> noop
Just k -> do
topf <- inRepo $ toTopFilePath f
Database.Keys.runWriter $ \h -> liftIO $ do
Database.Keys.SQL.addAssociatedFileFast k topf h
Database.Keys.SQL.addInodeCaches k [ic] h
liftIO $ void cleanup
Database.Keys.closeDb

View file

@ -1017,6 +1017,7 @@ Executable git-annex
Upgrade.V5
Upgrade.V5.Direct
Upgrade.V6
Upgrade.V7
Utility.Aeson
Utility.Android
Utility.Applicative