From 8a3b6f46c8da59de507e40e9dc0da175faba7637 Mon Sep 17 00:00:00 2001 From: yarikoptic Date: Fri, 3 Dec 2021 21:41:53 +0000 Subject: [PATCH 1/7] Added a comment --- .../comment_7_51f6745c4abce7b3cdfcf1c5998f8329._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_7_51f6745c4abce7b3cdfcf1c5998f8329._comment diff --git a/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_7_51f6745c4abce7b3cdfcf1c5998f8329._comment b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_7_51f6745c4abce7b3cdfcf1c5998f8329._comment new file mode 100644 index 0000000000..fbdadc2375 --- /dev/null +++ b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_7_51f6745c4abce7b3cdfcf1c5998f8329._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="yarikoptic" + avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4" + subject="comment 7" + date="2021-12-03T21:41:53Z" + content=""" +Great - thank you! FWIW I gave `8.20211123+git32-g9c0f3d1de-1~ndall+1_amd64` build a test run on that system, and it performed nicely (did get all files in my limited `get -J5` attempt on that dataset). Will be waiting for the ultimate solution to try. +"""]] From 9d3dce94f0af0564f8736aa16947300814662d12 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 3 Dec 2021 18:41:34 -0400 Subject: [PATCH 2/7] comment --- ..._d8ccfe3fe0d5123c1065ace73b623b44._comment | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_7_d8ccfe3fe0d5123c1065ace73b623b44._comment diff --git a/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_7_d8ccfe3fe0d5123c1065ace73b623b44._comment b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_7_d8ccfe3fe0d5123c1065ace73b623b44._comment new file mode 100644 index 0000000000..36bc79daa6 --- /dev/null +++ b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_7_d8ccfe3fe0d5123c1065ace73b623b44._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 7""" + date="2021-12-03T22:35:16Z" + content=""" +Unfortunately the more I dig into pid locking, the more deep problems I'm +finding.. + +When the pid lock's LockHandle is dropped, it drops the pid lock, and that +happens even if there is another LockHandle for the pid lock, which is +possble since it's a shared lock. So the pid lock may go away despite +a thread is still operating as if it is present. I think probably the pid +lock needs to stop using the LockPool and use a single LockHandle, which is +ref counted. + +`getLockStatus` and `checkSaneLock` look at the status of the pid lock, +but not yet at the fine-grained STM lock status. And as implemented, +I think they never worked at all, since they check for posix locks on the +pid lock file. +"""]] From e464ffd6413e4e5b72552e06f57517553385e78e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 3 Dec 2021 18:41:51 -0400 Subject: [PATCH 3/7] update comment to current status --- Utility/LockFile/PidLock.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Utility/LockFile/PidLock.hs b/Utility/LockFile/PidLock.hs index 786b961572..2646de94a8 100644 --- a/Utility/LockFile/PidLock.hs +++ b/Utility/LockFile/PidLock.hs @@ -279,7 +279,7 @@ waitedLock (Seconds timeout) lockfile displaymessage = do giveup $ "Gave up waiting for pid lock file " ++ fromRawFilePath lockfile -- | Use when the pid lock has already been taken by another thread of the --- same process, or perhaps is in the process of being taken. +-- same process. alreadyLocked :: MonadIO m => LockFile -> m LockHandle alreadyLocked lockfile = liftIO $ do abslockfile <- absPath lockfile From 00625caf873719708fd5b626d40b35ca585f5070 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 5 Dec 2021 08:11:41 -0400 Subject: [PATCH 4/7] update --- doc/thanks/list | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/thanks/list b/doc/thanks/list index 1876e7f234..9a57d462c5 100644 --- a/doc/thanks/list +++ b/doc/thanks/list @@ -121,3 +121,4 @@ Markus, Nick, Nicholas Golder-Manning, Troels Henriksen, +Max Thoursie, From a8ff96864ea3990982ae91daabc5d550fa4b4b42 Mon Sep 17 00:00:00 2001 From: "jasonb@ab4484d9961a46440958fa1a528e0fc435599057" Date: Sun, 5 Dec 2021 20:37:48 +0000 Subject: [PATCH 5/7] --- .../Use_assistant_to_sync_just_git_repos__63__.mdwn | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/forum/Use_assistant_to_sync_just_git_repos__63__.mdwn diff --git a/doc/forum/Use_assistant_to_sync_just_git_repos__63__.mdwn b/doc/forum/Use_assistant_to_sync_just_git_repos__63__.mdwn new file mode 100644 index 0000000000..db4efbbe1b --- /dev/null +++ b/doc/forum/Use_assistant_to_sync_just_git_repos__63__.mdwn @@ -0,0 +1,11 @@ +Hi, + +So lately I've been using git-annex to sync repos with only markdown files for Obsidian. Running `git annex sync --no-content` works well for this. But I have to remember when I've updated my Obsidian files on one system and make sure to sync to other repos. + +Could I use the assistant for this? For repos with annexed files, I've always preferred to manually manage my repos, so I've never used the assistant. + +Ideally, as I commit changes, it tries to sync them to other systems, and if those systems are offline (my MBP is closed for example and the ssh connection therefore times out), nothing explodes. + +Is this possible? Is it possible with a half dozen similar repos? Can a single agent do this or do I need to start one for each repo? + +Thanks! From 371238eca383d384cae3d5085e02cb97192bd1d1 Mon Sep 17 00:00:00 2001 From: alt Date: Mon, 6 Dec 2021 15:56:56 +0000 Subject: [PATCH 6/7] --- doc/forum/conflict_with_git-crypt.mdwn | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/forum/conflict_with_git-crypt.mdwn diff --git a/doc/forum/conflict_with_git-crypt.mdwn b/doc/forum/conflict_with_git-crypt.mdwn new file mode 100644 index 0000000000..0484ea5c87 --- /dev/null +++ b/doc/forum/conflict_with_git-crypt.mdwn @@ -0,0 +1,3 @@ +[git-crypt](https://www.agwa.name/projects/git-crypt/) is executed as a filter for secrets specified in `.gitattributes` files. However, git-annex initialization results in `* filter=annex` being added to `$GIT_DIR/info/attributes`, which [has the highest precedence](https://www.git-scm.com/docs/gitattributes#_description), ultimately causing any git-crypt filtering to be bypassed. + +Is there a recommended workaround for this? Thanks! From ae4c56b28adbe0423f1f7b3f3df4c2e9f7ec1625 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 6 Dec 2021 12:48:37 -0400 Subject: [PATCH 7/7] Revert "fix too early close of shared lock file" This reverts commit 66b2536ea0aa5c88f5b744eeace3322a8a4a10b6. I misunderstood commit ac56a5c2a056d3df1314a0a0cc16d276ddefb661 and caused a FD leak when pid locking is not used. A LockHandle contains an action that will close the underlying lock file, and that action is run when it is closed. In the case of a shared lock, the lock file is opened once for each LockHandle, and only the one for the LockHandle that is being closed will be closed. --- CHANGELOG | 1 - Utility/LockPool/STM.hs | 15 +++++++-------- ...nt_9_df98c47016e79f15218880a009e671ea._comment | 14 ++++++++++++++ 3 files changed, 21 insertions(+), 9 deletions(-) create mode 100644 doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_9_df98c47016e79f15218880a009e671ea._comment diff --git a/CHANGELOG b/CHANGELOG index cfb5f6092c..69d2da9f24 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -6,7 +6,6 @@ git-annex (8.20211124) UNRELEASED; urgency=medium erroring out. * export: Avoid unncessarily re-exporting non-annexed files that were already exported. - * Fix locking bug introduced in version 8.20200814. * Fix locking problems when annex.pidlock is set and concurrency is enabled eg with -J. diff --git a/Utility/LockPool/STM.hs b/Utility/LockPool/STM.hs index d5ab6b4e41..0d54d3dd06 100644 --- a/Utility/LockPool/STM.hs +++ b/Utility/LockPool/STM.hs @@ -28,7 +28,6 @@ import System.FilePath.ByteString (RawFilePath) import qualified Data.Map.Strict as M import Control.Concurrent.STM import Control.Exception -import Control.Monad type LockFile = RawFilePath @@ -129,8 +128,8 @@ getLockStatus pool file getdefault checker = do Nothing -> getdefault Just restore -> bracket_ (return ()) restore checker --- Only runs action to close underlying lock file when this is the last --- user of the lock, and when the lock has not already been closed. +-- Releases the lock. When it is a shared lock, it may remain locked by +-- other LockHandles. -- -- Note that the lock pool is left empty while the CloseLockFile action -- is run, to avoid race with another thread trying to open the same lock @@ -139,15 +138,15 @@ releaseLock :: LockHandle -> IO () releaseLock h = go =<< atomically (tryTakeTMVar h) where go (Just (pool, file, closelockfile)) = do - (m, lastuser) <- atomically $ do + m <- atomically $ do m <- takeTMVar pool return $ case M.lookup file m of Just (LockStatus mode n firstlocksem) - | n == 1 -> (M.delete file m, True) + | n == 1 -> (M.delete file m) | otherwise -> - (M.insert file (LockStatus mode (pred n) firstlocksem) m, False) - Nothing -> (m, True) - () <- when lastuser closelockfile + (M.insert file (LockStatus mode (pred n) firstlocksem) m) + Nothing -> m + () <- closelockfile atomically $ putTMVar pool m -- The LockHandle was already closed. go Nothing = return () diff --git a/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_9_df98c47016e79f15218880a009e671ea._comment b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_9_df98c47016e79f15218880a009e671ea._comment new file mode 100644 index 0000000000..cba3c4624c --- /dev/null +++ b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_9_df98c47016e79f15218880a009e671ea._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 9""" + date="2021-12-06T16:36:45Z" + content=""" +Unfortunately, [[!commit 66b2536ea0aa5c88f5b744eeace3322a8a4a10b6]] +broke shared locking when pid locks are not used, causing a FD leak, +and had to be reverted. Which further breaks concurrent pid locking. + +At this point, when -J is used with pidlock, it may drop the lock but +still think it has it held internally. Comment #7 found another way that +could happen, and the conclusion is that the pid lock must migrate away +from using the lock pool. +"""]]