From 879f52a11628441d65de56f82205aaff5509ea32 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 14 Feb 2020 15:22:48 -0400 Subject: [PATCH] annex.tune.branchhash1=true bugfix Fix support for repositories tuned with annex.tune.branchhash1=true, including --all not working and git-annex log not displaying anything for annexed files. --- Annex/Branch.hs | 3 +- Annex/Branch/Transitions.hs | 5 +-- CHANGELOG | 3 ++ Command/Log.hs | 9 +++--- Logs.hs | 31 ++++++++++++------- Logs/Location.hs | 6 ++-- Upgrade/V2.hs | 7 +++-- ...with_annex.tune.branchhash1__61__true.mdwn | 2 ++ ..._0e3da1c92dd46e4fc7facd10050fb590._comment | 18 +++++++++++ 9 files changed, 61 insertions(+), 23 deletions(-) create mode 100644 doc/bugs/git-annex_does_not_operate_on_all_keys_in_tuned_repository_with_annex.tune.branchhash1__61__true/comment_1_0e3da1c92dd46e4fc7facd10050fb590._comment diff --git a/Annex/Branch.hs b/Annex/Branch.hs index 6934e62bab..91d0276daa 100644 --- a/Annex/Branch.hs +++ b/Annex/Branch.hs @@ -577,10 +577,11 @@ performTransitionsLocked jl ts neednewlocalbranch transitionedrefs = do -} run [] = noop run changers = do + config <- Annex.getGitConfig trustmap <- calcTrustMap <$> getStaged trustLog remoteconfigmap <- calcRemoteConfigMap <$> getStaged remoteLog -- partially apply, improves performance - let changers' = map (\c -> c trustmap remoteconfigmap) changers + let changers' = map (\c -> c config trustmap remoteconfigmap) changers fs <- branchFiles forM_ fs $ \f -> do content <- getStaged f diff --git a/Annex/Branch/Transitions.hs b/Annex/Branch/Transitions.hs index 3041d54b3b..72c593c114 100644 --- a/Annex/Branch/Transitions.hs +++ b/Annex/Branch/Transitions.hs @@ -22,6 +22,7 @@ import Types.TrustLevel import Types.UUID import Types.MetaData import Types.Remote +import Types.GitConfig import Types.ProposedAccepted import Annex.SpecialRemote.Config @@ -35,7 +36,7 @@ data FileTransition = ChangeFile Builder | PreserveFile -type TransitionCalculator = TrustMap -> M.Map UUID RemoteConfig -> RawFilePath -> L.ByteString -> FileTransition +type TransitionCalculator = GitConfig -> TrustMap -> M.Map UUID RemoteConfig -> RawFilePath -> L.ByteString -> FileTransition getTransitionCalculator :: Transition -> Maybe TransitionCalculator getTransitionCalculator ForgetGitHistory = Nothing @@ -54,7 +55,7 @@ getTransitionCalculator ForgetDeadRemotes = Just dropDead -- is not removed from the remote log, for the same reason the trust log -- is not changed. dropDead :: TransitionCalculator -dropDead trustmap remoteconfigmap f content = case getLogVariety f of +dropDead config trustmap remoteconfigmap f content = case getLogVariety config f of Just OldUUIDBasedLog | f == trustLog -> PreserveFile | f == remoteLog -> ChangeFile $ diff --git a/CHANGELOG b/CHANGELOG index 227f74805e..4f442ca958 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -9,6 +9,9 @@ git-annex (7.20200205) UNRELEASED; urgency=medium be used, to clearly state why. * Avoid throwing fatal errors when asked to write to a readonly git remote on http. + * Fix support for repositories tuned with annex.tune.branchhash1=true, + including --all not working and git-annex log not displaying anything + for annexed files. -- Joey Hess Fri, 14 Feb 2020 14:12:25 -0400 diff --git a/Command/Log.hs b/Command/Log.hs index 397e6068d3..5597bfbf47 100644 --- a/Command/Log.hs +++ b/Command/Log.hs @@ -210,6 +210,7 @@ getAllLog = getGitLog [] getGitLog :: [FilePath] -> [CommandParam] -> Annex ([RefChange], IO Bool) getGitLog fs os = do + config <- Annex.getGitConfig (ls, cleanup) <- inRepo $ pipeNullSplit $ [ Param "log" , Param "-z" @@ -220,7 +221,7 @@ getGitLog fs os = do [ Param $ Git.fromRef Annex.Branch.fullname , Param "--" ] ++ map Param fs - return (parseGitRawLog (map decodeBL' ls), cleanup) + return (parseGitRawLog config (map decodeBL' ls), cleanup) -- Parses chunked git log --raw output, which looks something like: -- @@ -236,8 +237,8 @@ getGitLog fs os = do -- -- The timestamp is not included before all changelines, so -- keep track of the most recently seen timestamp. -parseGitRawLog :: [String] -> [RefChange] -parseGitRawLog = parse epoch +parseGitRawLog :: GitConfig -> [String] -> [RefChange] +parseGitRawLog config = parse epoch where epoch = toEnum 0 :: POSIXTime parse oldts ([]:rest) = parse oldts rest @@ -250,7 +251,7 @@ parseGitRawLog = parse epoch (tss, cl') -> (parseTimeStamp tss, cl') mrc = do (old, new) <- parseRawChangeLine cl - key <- locationLogFileKey (toRawFilePath c2) + key <- locationLogFileKey config (toRawFilePath c2) return $ RefChange { changetime = ts , oldref = old diff --git a/Logs.hs b/Logs.hs index 5faec561ef..906f406793 100644 --- a/Logs.hs +++ b/Logs.hs @@ -1,6 +1,6 @@ {- git-annex log file names - - - Copyright 2013-2019 Joey Hess + - Copyright 2013-2020 Joey Hess - - Licensed under the GNU AGPL version 3 or higher. -} @@ -27,8 +27,8 @@ data LogVariety {- Converts a path from the git-annex branch into one of the varieties - of logs used by git-annex, if it's a known path. -} -getLogVariety :: RawFilePath -> Maybe LogVariety -getLogVariety f +getLogVariety :: GitConfig -> RawFilePath -> Maybe LogVariety +getLogVariety config f | f `elem` topLevelOldUUIDBasedLogs = Just OldUUIDBasedLog | f `elem` topLevelNewUUIDBasedLogs = Just NewUUIDBasedLog | isRemoteStateLog f = Just NewUUIDBasedLog @@ -36,7 +36,7 @@ getLogVariety f | isChunkLog f = ChunkLog <$> extLogFileKey chunkLogExt f | isRemoteMetaDataLog f = Just RemoteMetaDataLog | isMetaDataLog f || f `elem` otherLogs = Just OtherLog - | otherwise = PresenceLog <$> firstJust (presenceLogs f) + | otherwise = PresenceLog <$> firstJust (presenceLogs config f) {- All the old-format uuid-based logs stored in the top of the git-annex branch. -} topLevelOldUUIDBasedLogs :: [RawFilePath] @@ -61,10 +61,10 @@ topLevelNewUUIDBasedLogs = {- All the ways to get a key from a presence log file -} -presenceLogs :: RawFilePath -> [Maybe Key] -presenceLogs f = +presenceLogs :: GitConfig -> RawFilePath -> [Maybe Key] +presenceLogs config f = [ urlLogFileKey f - , locationLogFileKey f + , locationLogFileKey config f ] {- Top-level logs that are neither UUID based nor presence logs. -} @@ -218,8 +218,17 @@ urlLogFileKey :: RawFilePath -> Maybe Key urlLogFileKey = extLogFileKey urlLogExt {- Converts a pathname into a key if it's a location log. -} -locationLogFileKey :: RawFilePath -> Maybe Key -locationLogFileKey path - -- Want only xx/yy/foo.log, not .log files in other places. - | length (splitDirectories (fromRawFilePath path)) /= 3 = Nothing +locationLogFileKey :: GitConfig -> RawFilePath -> Maybe Key +locationLogFileKey config path + | length (splitDirectories (fromRawFilePath path)) /= locationLogFileDepth config = Nothing | otherwise = extLogFileKey ".log" path + +{- Depth of location log files within the git-annex branch. + - + - Normally they are xx/yy/key.log so depth 3. + - The same extension is also used for other logs that + - are not location logs. -} +locationLogFileDepth :: GitConfig -> Int +locationLogFileDepth config = hashlevels + 1 + where + HashLevels hashlevels = branchHashLevels config diff --git a/Logs/Location.hs b/Logs/Location.hs index 66532ae413..2c3439805f 100644 --- a/Logs/Location.hs +++ b/Logs/Location.hs @@ -130,8 +130,10 @@ loggedKeys :: Annex [Unchecked Key] loggedKeys = loggedKeys' (not <$$> checkDead) loggedKeys' :: (Key -> Annex Bool) -> Annex [Unchecked Key] -loggedKeys' check = mapMaybe (defercheck <$$> locationLogFileKey) - <$> Annex.Branch.files +loggedKeys' check = do + config <- Annex.getGitConfig + mapMaybe (defercheck <$$> locationLogFileKey config) + <$> Annex.Branch.files where defercheck k = Unchecked $ ifM (check k) ( return (Just k) diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs index e255403d58..4a70a05d1a 100644 --- a/Upgrade/V2.hs +++ b/Upgrade/V2.hs @@ -67,16 +67,17 @@ upgrade = do locationLogs :: Annex [(Key, FilePath)] locationLogs = do + config <- Annex.getGitConfig dir <- fromRepo gitStateDir liftIO $ do levela <- dirContents dir levelb <- mapM tryDirContents levela files <- mapM tryDirContents (concat levelb) - return $ mapMaybe islogfile (concat files) + return $ mapMaybe (islogfile config) (concat files) where tryDirContents d = catchDefaultIO [] $ dirContents d - islogfile f = maybe Nothing (\k -> Just (k, f)) $ - locationLogFileKey (toRawFilePath f) + islogfile config f = maybe Nothing (\k -> Just (k, f)) $ + locationLogFileKey config (toRawFilePath f) inject :: FilePath -> FilePath -> Annex () inject source dest = do diff --git a/doc/bugs/git-annex_does_not_operate_on_all_keys_in_tuned_repository_with_annex.tune.branchhash1__61__true.mdwn b/doc/bugs/git-annex_does_not_operate_on_all_keys_in_tuned_repository_with_annex.tune.branchhash1__61__true.mdwn index 9467c8ced6..7c10599b43 100644 --- a/doc/bugs/git-annex_does_not_operate_on_all_keys_in_tuned_repository_with_annex.tune.branchhash1__61__true.mdwn +++ b/doc/bugs/git-annex_does_not_operate_on_all_keys_in_tuned_repository_with_annex.tune.branchhash1__61__true.mdwn @@ -87,3 +87,5 @@ Repository status: Of course, I love it! Great project, thanks, Joey! However, /me always wants more features from it. It's great that git-annex continues to develop. + +> [[fixed|done]] --[[Joey]] diff --git a/doc/bugs/git-annex_does_not_operate_on_all_keys_in_tuned_repository_with_annex.tune.branchhash1__61__true/comment_1_0e3da1c92dd46e4fc7facd10050fb590._comment b/doc/bugs/git-annex_does_not_operate_on_all_keys_in_tuned_repository_with_annex.tune.branchhash1__61__true/comment_1_0e3da1c92dd46e4fc7facd10050fb590._comment new file mode 100644 index 0000000000..9a9dd2fe56 --- /dev/null +++ b/doc/bugs/git-annex_does_not_operate_on_all_keys_in_tuned_repository_with_annex.tune.branchhash1__61__true/comment_1_0e3da1c92dd46e4fc7facd10050fb590._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="joey" + subject="""comment 1""" + date="2020-02-14T19:01:50Z" + content=""" +Indeed this is a bug. Easy to see why, note that "3": + + locationLogFileKey path + -- Want only xx/yy/foo.log, not .log files in other places. + | length (splitDirectories (fromRawFilePath path)) /= 3 = Nothing + +So this also affected some other things that use that. Including `git-annex log`, +potentially something to do with v2 upgrade (if a v2 repo could be tuned this way?), +and handling transitions set up by `git annex forget`. + +(I also checked if annex.tune.objecthash1 similarly broke stuff that +enumerated .git/annex/objects, but that was handled ok already.) +"""]]