diff --git a/Assistant/Threads/Committer.hs b/Assistant/Threads/Committer.hs index e10d3fd1a6..b7651917be 100644 --- a/Assistant/Threads/Committer.hs +++ b/Assistant/Threads/Committer.hs @@ -1,6 +1,6 @@ {- git-annex assistant commit thread - - - Copyright 2012, 2019 Joey Hess + - Copyright 2012-2021 Joey Hess - - Licensed under the GNU AGPL version 3 or higher. -} @@ -33,6 +33,7 @@ import Annex.Perms import Annex.CatFile import Annex.InodeSentinal import Annex.CurrentBranch +import Annex.FileMatcher import qualified Annex import Utility.InodeCache import qualified Database.Keys @@ -52,6 +53,7 @@ commitThread = namedThread "Committer" $ do havelsof <- liftIO $ inSearchPath "lsof" delayadd <- liftAnnex $ fmap Seconds . annexDelayAdd <$> Annex.getGitConfig + largefilematcher <- liftAnnex largeFilesMatcher msg <- liftAnnex Command.Sync.commitMsg lockdowndir <- liftAnnex $ fromRepo gitAnnexTmpWatcherDir liftAnnex $ do @@ -61,7 +63,7 @@ commitThread = namedThread "Committer" $ do (fromRawFilePath lockdowndir) void $ createAnnexDirectory lockdowndir waitChangeTime $ \(changes, time) -> do - readychanges <- handleAdds (fromRawFilePath lockdowndir) havelsof delayadd $ + readychanges <- handleAdds (fromRawFilePath lockdowndir) havelsof largefilematcher delayadd $ simplifyChanges changes if shouldCommit False time (length readychanges) readychanges then do @@ -239,7 +241,7 @@ commitStaged msg = do return ok {- If there are PendingAddChanges, or InProcessAddChanges, the files - - have not yet actually been added to the annex, and that has to be done + - have not yet actually been added, and that has to be done - now, before committing. - - Deferring the adds to this point causes batches to be bundled together, @@ -257,8 +259,8 @@ commitStaged msg = do - Any pending adds that are not ready yet are put back into the ChangeChan, - where they will be retried later. -} -handleAdds :: FilePath -> Bool -> Maybe Seconds -> [Change] -> Assistant [Change] -handleAdds lockdowndir havelsof delayadd cs = returnWhen (null incomplete) $ do +handleAdds :: FilePath -> Bool -> GetFileMatcher -> Maybe Seconds -> [Change] -> Assistant [Change] +handleAdds lockdowndir havelsof largefilematcher delayadd cs = returnWhen (null incomplete) $ do let (pending, inprocess) = partition isPendingAddChange incomplete let lockdownconfig = LockDownConfig { lockingFile = False @@ -271,9 +273,12 @@ handleAdds lockdowndir havelsof delayadd cs = returnWhen (null incomplete) $ do refillChanges postponed returnWhen (null toadd) $ do - added <- addaction toadd $ - catMaybes <$> addunlocked toadd - return $ added ++ otherchanges + (toaddannexed, toaddsmall) <- partitionEithers + <$> mapM checksmall toadd + addsmall toaddsmall + addedannexed <- addaction toadd $ + catMaybes <$> addannexed toaddannexed + return $ addedannexed ++ toaddsmall ++ otherchanges where (incomplete, otherchanges) = partition (\c -> isPendingAddChange c || isInProcessAddChange c) cs @@ -281,25 +286,24 @@ handleAdds lockdowndir havelsof delayadd cs = returnWhen (null incomplete) $ do | c = return otherchanges | otherwise = a - add :: LockDownConfig -> Change -> Assistant (Maybe Change) - add lockdownconfig change@(InProcessAddChange { lockedDown = ld }) = - catchDefaultIO Nothing <~> doadd - where - ks = keySource ld - doadd = sanitycheck ks $ do - (mkey, _mcache) <- liftAnnex $ do - showStart "add" (keyFilename ks) (SeekInput []) - ingest nullMeterUpdate (Just $ LockedDown lockdownconfig ks) Nothing - maybe (failedingest change) (done change $ fromRawFilePath $ keyFilename ks) mkey - add _ _ = return Nothing + checksmall change = + ifM (liftAnnex $ checkFileMatcher largefilematcher (toRawFilePath (changeFile change))) + ( return (Left change) + , return (Right change) + ) + + addsmall [] = noop + addsmall toadd = liftAnnex $ Annex.Queue.addCommand [] "add" + [ Param "--force", Param "--"] (map changeFile toadd) {- Avoid overhead of re-injesting a renamed unlocked file, by - examining the other Changes to see if a removed file has the - same InodeCache as the new file. If so, we can just update - bookkeeping, and stage the file in git. -} - addunlocked :: [Change] -> Assistant [Maybe Change] - addunlocked toadd = do + addannexed :: [Change] -> Assistant [Maybe Change] + addannexed [] = return [] + addannexed toadd = do ct <- liftAnnex compareInodeCachesWith m <- liftAnnex $ removedKeysMap ct cs delta <- liftAnnex getTSDelta @@ -308,16 +312,28 @@ handleAdds lockdowndir havelsof delayadd cs = returnWhen (null incomplete) $ do , hardlinkFileTmpDir = Just (toRawFilePath lockdowndir) } if M.null m - then forM toadd (add cfg) + then forM toadd (addannexed' cfg) else forM toadd $ \c -> do mcache <- liftIO $ genInodeCache (toRawFilePath (changeFile c)) delta case mcache of - Nothing -> add cfg c + Nothing -> addannexed' cfg c Just cache -> case M.lookup (inodeCacheToKey ct cache) m of - Nothing -> add cfg c + Nothing -> addannexed' cfg c Just k -> fastadd c k + addannexed' :: LockDownConfig -> Change -> Assistant (Maybe Change) + addannexed' lockdownconfig change@(InProcessAddChange { lockedDown = ld }) = + catchDefaultIO Nothing <~> doadd + where + ks = keySource ld + doadd = sanitycheck ks $ do + (mkey, _mcache) <- liftAnnex $ do + showStart "add" (keyFilename ks) (SeekInput []) + ingest nullMeterUpdate (Just $ LockedDown lockdownconfig ks) Nothing + maybe (failedingest change) (done change $ fromRawFilePath $ keyFilename ks) mkey + addannexed' _ _ = return Nothing + fastadd :: Change -> Key -> Assistant (Maybe Change) fastadd change key = do let source = keySource $ lockedDown change diff --git a/Assistant/Threads/Watcher.hs b/Assistant/Threads/Watcher.hs index c617fb6c6b..72ff1233a4 100644 --- a/Assistant/Threads/Watcher.hs +++ b/Assistant/Threads/Watcher.hs @@ -34,7 +34,6 @@ import Annex.WorkTree import Annex.CatFile import Annex.CheckIgnore import Annex.Link -import Annex.FileMatcher import Annex.Content import Annex.ReplaceFile import Annex.InodeSentinal @@ -89,9 +88,8 @@ watchThread = namedThread "Watcher" $ runWatcher :: Assistant () runWatcher = do startup <- asIO1 startupScan - matcher <- liftAnnex largeFilesMatcher symlinkssupported <- liftAnnex $ coreSymlinks <$> Annex.getGitConfig - addhook <- hook $ onAddUnlocked symlinkssupported matcher + addhook <- hook $ onAddUnlocked symlinkssupported delhook <- hook onDel addsymlinkhook <- hook onAddSymlink deldirhook <- hook onDelDir @@ -193,24 +191,14 @@ runHandler handler file filestatus = void $ do | "./" `isPrefixOf` file = drop 2 f | otherwise = f -{- Small files are added to git as-is, while large ones go into the annex. -} -add :: GetFileMatcher -> FilePath -> Assistant (Maybe Change) -add largefilematcher file = ifM (liftAnnex $ checkFileMatcher largefilematcher (toRawFilePath file)) - ( pendingAddChange file - , do - liftAnnex $ Annex.Queue.addCommand [] "add" - [Param "--force", Param "--"] [file] - madeChange file AddFileChange - ) - shouldRestage :: DaemonStatus -> Bool shouldRestage ds = scanComplete ds || forceRestage ds -onAddUnlocked :: Bool -> GetFileMatcher -> Handler -onAddUnlocked symlinkssupported matcher f fs = do +onAddUnlocked :: Bool -> Handler +onAddUnlocked symlinkssupported f fs = do mk <- liftIO $ isPointerFile $ toRawFilePath f case mk of - Nothing -> onAddUnlocked' contentchanged addassociatedfile addlink samefilestatus symlinkssupported matcher f fs + Nothing -> onAddUnlocked' contentchanged addassociatedfile addlink samefilestatus symlinkssupported f fs Just k -> addlink f k where addassociatedfile key file = @@ -240,9 +228,8 @@ onAddUnlocked' -> (FilePath -> Key -> Assistant (Maybe Change)) -> (Key -> FilePath -> FileStatus -> Annex Bool) -> Bool - -> GetFileMatcher -> Handler -onAddUnlocked' contentchanged addassociatedfile addlink samefilestatus symlinkssupported matcher file fs = do +onAddUnlocked' contentchanged addassociatedfile addlink samefilestatus symlinkssupported file fs = do v <- liftAnnex $ catKeyFile (toRawFilePath file) case (v, fs) of (Just key, Just filestatus) -> @@ -259,12 +246,12 @@ onAddUnlocked' contentchanged addassociatedfile addlink samefilestatus symlinkss , guardSymlinkStandin (Just key) $ do debug ["changed", file] liftAnnex $ contentchanged key file - add matcher file + pendingAddChange file ) _ -> unlessIgnored file $ guardSymlinkStandin Nothing $ do debug ["add", file] - add matcher file + pendingAddChange file where {- On a filesystem without symlinks, we'll get changes for regular - files that git uses to stand-in for symlinks. Detect when diff --git a/CHANGELOG b/CHANGELOG index 1258118818..bedcafa6a0 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -8,6 +8,9 @@ git-annex (8.20210631) UNRELEASED; urgency=medium * init: Fix misbehavior when core.sharedRepository = group that caused it to enter an adjusted branch and set annex.crippledfilesystem (Reversion in version 8.20210630) + * assistant: When adding non-large files to git, honor annex.delayadd + configuration. Also, don't add non-large files to git when they + are still being written to. -- Joey Hess Wed, 30 Jun 2021 17:55:10 -0400 diff --git a/doc/bugs/delayadd_doesn__39__t_work_with_smallfiles.mdwn b/doc/bugs/delayadd_doesn__39__t_work_with_smallfiles.mdwn index fa140ffb0c..4068a55905 100644 --- a/doc/bugs/delayadd_doesn__39__t_work_with_smallfiles.mdwn +++ b/doc/bugs/delayadd_doesn__39__t_work_with_smallfiles.mdwn @@ -42,3 +42,5 @@ local repository version: 8 """]] ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders) + +> [[fixed|done]] --[[Joey]] diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 4c2fa4ff3a..d6fceb647c 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -1765,8 +1765,8 @@ Remotes are configured using these settings in `.git/config`. to close it. Note that this only delays adding files created while the daemon is - running. Changes made when it is not running will be added the next time - it is started up. + running. Changes made when it is not running will be added immediately + the next time it is started up. * `annex.expireunused`