diff --git a/CHANGELOG b/CHANGELOG index cfb5f6092c..69d2da9f24 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -6,7 +6,6 @@ git-annex (8.20211124) UNRELEASED; urgency=medium erroring out. * export: Avoid unncessarily re-exporting non-annexed files that were already exported. - * Fix locking bug introduced in version 8.20200814. * Fix locking problems when annex.pidlock is set and concurrency is enabled eg with -J. diff --git a/Utility/LockFile/PidLock.hs b/Utility/LockFile/PidLock.hs index 83541b7b94..38795f8810 100644 --- a/Utility/LockFile/PidLock.hs +++ b/Utility/LockFile/PidLock.hs @@ -280,7 +280,7 @@ waitedLock (Seconds timeout) lockfile displaymessage = do giveup $ "Gave up waiting for pid lock file " ++ fromRawFilePath lockfile -- | Use when the pid lock has already been taken by another thread of the --- same process, or perhaps is in the process of being taken. +-- same process. alreadyLocked :: MonadIO m => PidLockFile -> m LockHandle alreadyLocked lockfile = liftIO $ do abslockfile <- absPath lockfile diff --git a/Utility/LockPool/STM.hs b/Utility/LockPool/STM.hs index 84c68faf3e..37bcf4f192 100644 --- a/Utility/LockPool/STM.hs +++ b/Utility/LockPool/STM.hs @@ -29,7 +29,6 @@ import System.FilePath.ByteString (RawFilePath) import qualified Data.Map.Strict as M import Control.Concurrent.STM import Control.Exception -import Control.Monad type LockFile = RawFilePath @@ -142,8 +141,8 @@ getLockStatus pool file getdefault checker = do Nothing -> getdefault Just restore -> bracket_ (return ()) restore checker --- Only runs action to close underlying lock file when this is the last --- user of the lock, and when the lock has not already been closed. +-- Releases the lock. When it is a shared lock, it may remain locked by +-- other LockHandles. -- -- Note that the lock pool is left empty while the CloseLockFile action -- is run, to avoid race with another thread trying to open the same lock @@ -153,16 +152,16 @@ releaseLock :: LockHandle -> IO () releaseLock h = go =<< atomically (tryTakeTMVar h) where go (Just (pool, file, closelockfile, postreleaselock)) = do - (m, lastuser) <- atomically $ do + m <- atomically $ do m <- takeTMVar pool return $ case M.lookup file m of Just (LockStatus mode n firstlocksem) - | n == 1 -> (M.delete file m, True) + | n == 1 -> (M.delete file m) | otherwise -> - (M.insert file (LockStatus mode (pred n) firstlocksem) m, False) - Nothing -> (m, True) - when lastuser closelockfile + (M.insert file (LockStatus mode (pred n) firstlocksem) m) + Nothing -> m + () <- closelockfile atomically $ putTMVar pool m - when lastuser postreleaselock + postreleaselock -- The LockHandle was already closed. go Nothing = return () diff --git a/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_7_51f6745c4abce7b3cdfcf1c5998f8329._comment b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_7_51f6745c4abce7b3cdfcf1c5998f8329._comment new file mode 100644 index 0000000000..fbdadc2375 --- /dev/null +++ b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_7_51f6745c4abce7b3cdfcf1c5998f8329._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="comment 7" + date="2021-12-03T21:41:53Z" + content=""" +Great - thank you! FWIW I gave `8.20211123+git32-g9c0f3d1de-1~ndall+1_amd64` build a test run on that system, and it performed nicely (did get all files in my limited `get -J5` attempt on that dataset). Will be waiting for the ultimate solution to try. +"""]] diff --git a/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_7_d8ccfe3fe0d5123c1065ace73b623b44._comment b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_7_d8ccfe3fe0d5123c1065ace73b623b44._comment new file mode 100644 index 0000000000..36bc79daa6 --- /dev/null +++ b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_7_d8ccfe3fe0d5123c1065ace73b623b44._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 7""" + date="2021-12-03T22:35:16Z" + content=""" +Unfortunately the more I dig into pid locking, the more deep problems I'm +finding.. + +When the pid lock's LockHandle is dropped, it drops the pid lock, and that +happens even if there is another LockHandle for the pid lock, which is +possble since it's a shared lock. So the pid lock may go away despite +a thread is still operating as if it is present. I think probably the pid +lock needs to stop using the LockPool and use a single LockHandle, which is +ref counted. + +`getLockStatus` and `checkSaneLock` look at the status of the pid lock, +but not yet at the fine-grained STM lock status. And as implemented, +I think they never worked at all, since they check for posix locks on the +pid lock file. +"""]] diff --git a/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_9_df98c47016e79f15218880a009e671ea._comment b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_9_df98c47016e79f15218880a009e671ea._comment new file mode 100644 index 0000000000..cba3c4624c --- /dev/null +++ b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_9_df98c47016e79f15218880a009e671ea._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 9""" + date="2021-12-06T16:36:45Z" + content=""" +Unfortunately, [[!commit 66b2536ea0aa5c88f5b744eeace3322a8a4a10b6]] +broke shared locking when pid locks are not used, causing a FD leak, +and had to be reverted. Which further breaks concurrent pid locking. + +At this point, when -J is used with pidlock, it may drop the lock but +still think it has it held internally. Comment #7 found another way that +could happen, and the conclusion is that the pid lock must migrate away +from using the lock pool. +"""]] diff --git a/doc/forum/Use_assistant_to_sync_just_git_repos__63__.mdwn b/doc/forum/Use_assistant_to_sync_just_git_repos__63__.mdwn new file mode 100644 index 0000000000..db4efbbe1b --- /dev/null +++ b/doc/forum/Use_assistant_to_sync_just_git_repos__63__.mdwn @@ -0,0 +1,11 @@ +Hi, + +So lately I've been using git-annex to sync repos with only markdown files for Obsidian. Running `git annex sync --no-content` works well for this. But I have to remember when I've updated my Obsidian files on one system and make sure to sync to other repos. + +Could I use the assistant for this? For repos with annexed files, I've always preferred to manually manage my repos, so I've never used the assistant. + +Ideally, as I commit changes, it tries to sync them to other systems, and if those systems are offline (my MBP is closed for example and the ssh connection therefore times out), nothing explodes. + +Is this possible? Is it possible with a half dozen similar repos? Can a single agent do this or do I need to start one for each repo? + +Thanks! diff --git a/doc/forum/conflict_with_git-crypt.mdwn b/doc/forum/conflict_with_git-crypt.mdwn new file mode 100644 index 0000000000..0484ea5c87 --- /dev/null +++ b/doc/forum/conflict_with_git-crypt.mdwn @@ -0,0 +1,3 @@ +[git-crypt](https://www.agwa.name/projects/git-crypt/) is executed as a filter for secrets specified in `.gitattributes` files. However, git-annex initialization results in `* filter=annex` being added to `$GIT_DIR/info/attributes`, which [has the highest precedence](https://www.git-scm.com/docs/gitattributes#_description), ultimately causing any git-crypt filtering to be bypassed. + +Is there a recommended workaround for this? Thanks! diff --git a/doc/thanks/list b/doc/thanks/list index 1876e7f234..9a57d462c5 100644 --- a/doc/thanks/list +++ b/doc/thanks/list @@ -121,3 +121,4 @@ Markus, Nick, Nicholas Golder-Manning, Troels Henriksen, +Max Thoursie,