diff --git a/Annex/WorkTree.hs b/Annex/WorkTree.hs index 9ac59d8eb0..f8ab63a1bb 100644 --- a/Annex/WorkTree.hs +++ b/Annex/WorkTree.hs @@ -1,6 +1,6 @@ {- git-annex worktree files - - - Copyright 2013-2019 Joey Hess + - Copyright 2013-2020 Joey Hess - - Licensed under the GNU AGPL version 3 or higher. -} @@ -24,6 +24,8 @@ import qualified Database.Keys.SQL import Config import qualified Utility.RawFilePath as R +import Control.Concurrent + {- Looks up the key corresponding to an annexed file in the work tree, - by examining what the file links to. - @@ -77,12 +79,13 @@ ifAnnexed file yes no = maybe no yes =<< lookupFile file -} scanUnlockedFiles :: Annex () scanUnlockedFiles = whenM (inRepo Git.Ref.headExists <&&> not <$> isBareRepo) $ do - Database.Keys.runWriter $ - liftIO . Database.Keys.SQL.dropAllAssociatedFiles + dropold <- liftIO $ newMVar $ + Database.Keys.runWriter $ + liftIO . Database.Keys.SQL.dropAllAssociatedFiles (l, cleanup) <- inRepo $ Git.LsTree.lsTree Git.LsTree.LsTreeRecursive Git.Ref.headRef forM_ l $ \i -> when (isregfile i) $ - maybe noop (add i) + maybe noop (add dropold i) =<< catKey (Git.LsTree.sha i) liftIO $ void cleanup where @@ -90,7 +93,8 @@ scanUnlockedFiles = whenM (inRepo Git.Ref.headExists <&&> not <$> isBareRepo) $ Just Git.Types.TreeFile -> True Just Git.Types.TreeExecutable -> True _ -> False - add i k = do + add dropold i k = do + join $ fromMaybe noop <$> liftIO (tryTakeMVar dropold) let tf = Git.LsTree.file i Database.Keys.runWriter $ liftIO . Database.Keys.SQL.addAssociatedFileFast k tf diff --git a/CHANGELOG b/CHANGELOG index ac742ff5ae..bfa04b351f 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -29,6 +29,9 @@ git-annex (8.20200523) UNRELEASED; urgency=medium that is configured with exporttree=yes. * Note that external special remote programs should not block SIGINT or SIGTERM. + * Avoid creating the keys database during init when there are no unlocked + files, to prevent init failing when sqlite does not work in the + filesystem. -- Joey Hess Tue, 26 May 2020 10:20:52 -0400 diff --git a/doc/bugs/crippled_fs___40__pidlock__41___leads_to_git-annex__58___SQLite3_returned_ErrorIO_while_attempting_to_perform_prepare___34__PRAGMA_journal__95__mode__61__WAL__59____34____58___disk_I__47__O_error/comment_1_aaea812a8f2b04d9b17d1ba9c6fad39f._comment b/doc/bugs/crippled_fs___40__pidlock__41___leads_to_git-annex__58___SQLite3_returned_ErrorIO_while_attempting_to_perform_prepare___34__PRAGMA_journal__95__mode__61__WAL__59____34____58___disk_I__47__O_error/comment_1_aaea812a8f2b04d9b17d1ba9c6fad39f._comment new file mode 100644 index 0000000000..e28bcb49fa --- /dev/null +++ b/doc/bugs/crippled_fs___40__pidlock__41___leads_to_git-annex__58___SQLite3_returned_ErrorIO_while_attempting_to_perform_prepare___34__PRAGMA_journal__95__mode__61__WAL__59____34____58___disk_I__47__O_error/comment_1_aaea812a8f2b04d9b17d1ba9c6fad39f._comment @@ -0,0 +1,60 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2020-06-11T18:13:05Z" + content=""" +This is sqlite failing to do its own internal locking which I'll bet is +some elaborate and fine-grained locking indeed. + +Essentially the same problem as +[[drop_blows_on_lustre__58___SQLite3_returned_ErrorIO]] +(And also the main problem using git-annex on Windows in WSL.) + +Not a great deal I can do about this, because the locking is in sqlite. +That other bug has some thoughts about avoiding unncessary use of sqlite +or moving the databases to some other filesystem. + +It may be that not enabling WAL mode on the database would change sqlite's +locking behavior to something that works better on the filesystem. +I would normally not want to do that, but with pidlock enabled, only one +git-annex is going to be running anyway, and so there's no need for WAL +mode's concurrency support. + +Here's a patch to test. I would not use this in production, since it does +not check for pidlock, but just always disables WAL. + + diff --git a/Database/Handle.hs b/Database/Handle.hs + index 4422d9e75..790833d72 100644 + --- a/Database/Handle.hs + +++ b/Database/Handle.hs + @@ -34,6 +34,7 @@ import qualified Data.Text as T + import Control.Monad.Trans.Resource (runResourceT) + import Control.Monad.Logger (runNoLoggingT) + import System.IO + +import Lens.Micro + + {- A DbHandle is a reference to a worker thread that communicates with + - the database. It has a MVar which Jobs are submitted to. -} + @@ -196,8 +197,9 @@ runSqliteRobustly tablename db a = do + rethrow msg e = throwIO $ userError $ show e ++ "(" ++ msg ++ ")" + + go conn retries = do + + let i = over walEnabled (const False) (mkSqliteConnectionInfo db) + r <- try $ runResourceT $ runNoLoggingT $ + - withSqlConnRobustly (wrapConnection conn) $ + + withSqlConnRobustly (wrapConnectionInfo i conn) $ + runSqlConn a + case r of + Right v -> return v + diff --git a/Database/Init.hs b/Database/Init.hs + index c66fe43ac..0c8478c7d 100644 + --- a/Database/Init.hs + +++ b/Database/Init.hs + @@ -52,5 +52,5 @@ initDb db migration = do + - Note that once WAL mode is enabled, it will persist whenever the + - database is opened. -} + enableWAL :: T.Text -> SqliteConnectionInfo + -enableWAL db = over walEnabled (const True) $ + +enableWAL db = over walEnabled (const False) $ + mkSqliteConnectionInfo db +"""]] diff --git a/doc/bugs/crippled_fs___40__pidlock__41___leads_to_git-annex__58___SQLite3_returned_ErrorIO_while_attempting_to_perform_prepare___34__PRAGMA_journal__95__mode__61__WAL__59____34____58___disk_I__47__O_error/comment_2_c55f2d95f7f08e6f6439f36790e5a538._comment b/doc/bugs/crippled_fs___40__pidlock__41___leads_to_git-annex__58___SQLite3_returned_ErrorIO_while_attempting_to_perform_prepare___34__PRAGMA_journal__95__mode__61__WAL__59____34____58___disk_I__47__O_error/comment_2_c55f2d95f7f08e6f6439f36790e5a538._comment new file mode 100644 index 0000000000..bf0b919197 --- /dev/null +++ b/doc/bugs/crippled_fs___40__pidlock__41___leads_to_git-annex__58___SQLite3_returned_ErrorIO_while_attempting_to_perform_prepare___34__PRAGMA_journal__95__mode__61__WAL__59____34____58___disk_I__47__O_error/comment_2_c55f2d95f7f08e6f6439f36790e5a538._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 2""" + date="2020-06-11T19:26:05Z" + content=""" +This does seem to be an unncessary use of the database, +git-annex init always creates the db in scanUnlockedFiles +when the repo has a HEAD ref, but normally it will find no unlocked files +and so it could just not create the database there. + +That should make git-annex usable, until a situation where it does +need to use the database, like something to do with an unlocked file. + +Implemented that, which will fix the immediate problem reported here, +but I'd like to pursue the WAL stuff still. +"""]]