From 86dbe9a825b9c615c63e0cfc5e4a737a249f8989 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Dec 2023 16:22:14 -0400 Subject: [PATCH] migrate: support adding size back to URL keys migrate: Support adding size to URL keys that were added with --relaxed, by running eg: git-annex migrate --backend=URL foo Since url keys cannot be generated, that used to fail. Make it notice that the backend is not changed, and just get the size of the content. Sponsored-by: Brock Spratlen on Patreon --- Annex/Content.hs | 25 ++++++++++++++++++- Annex/Content/Presence.hs | 3 ++- CHANGELOG | 2 ++ Command/Migrate.hs | 49 +++++++++++++++++++++----------------- doc/git-annex-migrate.mdwn | 4 ++++ 5 files changed, 59 insertions(+), 24 deletions(-) diff --git a/Annex/Content.hs b/Annex/Content.hs index c3766c0abc..04fc474d16 100644 --- a/Annex/Content.hs +++ b/Annex/Content.hs @@ -1,6 +1,6 @@ {- git-annex file content managing - - - Copyright 2010-2022 Joey Hess + - Copyright 2010-2023 Joey Hess - - Licensed under the GNU AGPL version 3 or higher. -} @@ -66,6 +66,7 @@ module Annex.Content ( getKeyStatus, getKeyFileStatus, cleanObjectDirs, + contentSize, ) where import System.IO.Unsafe (unsafeInterleaveIO) @@ -916,3 +917,25 @@ getKeyFileStatus key file = do ) _ -> return s +{- Gets the size of the content of a key when it is present. + - Useful when the key does not have keySize set. + - + - When the object file appears possibly modified with annex.thin set, does + - not do an expensive verification that the content is good, just returns + - Nothing. + -} +contentSize :: Key -> Annex (Maybe FileSize) +contentSize key = catchDefaultIO Nothing $ + withObjectLoc key $ \loc -> + withTSDelta (liftIO . genInodeCache loc) >>= \case + Just ic -> ifM (unmodified ic) + ( return (Just (inodeCacheFileSize ic)) + , return Nothing + ) + Nothing -> return Nothing + where + unmodified ic = + ifM (annexThin <$> Annex.getGitConfig) + ( isUnmodifiedCheap' key ic + , return True + ) diff --git a/Annex/Content/Presence.hs b/Annex/Content/Presence.hs index d3aea87151..2eb0016ddd 100644 --- a/Annex/Content/Presence.hs +++ b/Annex/Content/Presence.hs @@ -18,6 +18,7 @@ module Annex.Content.Presence ( isUnmodified, isUnmodified', isUnmodifiedCheap, + isUnmodifiedCheap', withContentLockFile, contentLockFile, ) where @@ -206,7 +207,7 @@ isUnmodified' = isUnmodifiedLowLevel Database.Keys.addInodeCaches - within a small time window (eg 1 second). -} isUnmodifiedCheap :: Key -> RawFilePath -> Annex Bool -isUnmodifiedCheap key f = maybe (return False) (isUnmodifiedCheap' key) +isUnmodifiedCheap key f = maybe (pure False) (isUnmodifiedCheap' key) =<< withTSDelta (liftIO . genInodeCache f) isUnmodifiedCheap' :: Key -> InodeCache -> Annex Bool diff --git a/CHANGELOG b/CHANGELOG index ba666913ad..862e594492 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -9,6 +9,8 @@ git-annex (10.20231130) UNRELEASED; urgency=medium that have been migrated. * Added annex.syncmigrations config that can be set to false to prevent pull and sync from migrating object content. + * migrate: Support adding size to URL keys that were added with + --relaxed, by running eg: git-annex migrate --backend=URL foo * Make git-annex get/copy/move --from foo override configuration of remote.foo.annex-ignore, as documented. * Support git-annex copy/move --from-anywhere --to remote. diff --git a/Command/Migrate.hs b/Command/Migrate.hs index 344287f506..c07377aae2 100644 --- a/Command/Migrate.hs +++ b/Command/Migrate.hs @@ -88,29 +88,26 @@ start o ksha si file key = do Just oldbackend -> do exists <- inAnnex key newbackend <- chooseBackend file - if (newbackend /= oldbackend || upgradableKey oldbackend key || forced) && exists + if (newbackend /= oldbackend || upgradableKey oldbackend || forced) && exists then go False oldbackend newbackend - else if removeSize o && exists - then go True oldbackend oldbackend + else if cantweaksize newbackend oldbackend && exists + then go True oldbackend newbackend else stop where - go onlyremovesize oldbackend newbackend = do + go onlytweaksize oldbackend newbackend = do keyrec <- case ksha of Just (KeySha s) -> pure (MigrationRecord s) Nothing -> error "internal" starting "migrate" (mkActionItem (key, file)) si $ - perform onlyremovesize o file key keyrec oldbackend newbackend + perform onlytweaksize o file key keyrec oldbackend newbackend -{- Checks if a key is upgradable to a newer representation. - - - - Reasons for migration: - - - Ideally, all keys have file size metadata. Old keys may not. - - - Something has changed in the backend, such as a bug fix. - -} -upgradableKey :: Backend -> Key -> Bool -upgradableKey backend key = isNothing (fromKey keySize key) || backendupgradable - where - backendupgradable = maybe False (\a -> a key) (canUpgradeKey backend) + cantweaksize newbackend oldbackend + | removeSize o = isJust (fromKey keySize key) + | newbackend /= oldbackend = False + | isNothing (fromKey keySize key) = True + | otherwise = False + + upgradableKey oldbackend = maybe False (\a -> a key) (canUpgradeKey oldbackend) {- Store the old backend's key in the new backend - The old backend's key is not dropped from it, because there may @@ -122,13 +119,13 @@ upgradableKey backend key = isNothing (fromKey keySize key) || backendupgradable - generated. -} perform :: Bool -> MigrateOptions -> RawFilePath -> Key -> MigrationRecord -> Backend -> Backend -> CommandPerform -perform onlyremovesize o file oldkey oldkeyrec oldbackend newbackend = go =<< genkey (fastMigrate oldbackend) +perform onlytweaksize o file oldkey oldkeyrec oldbackend newbackend = go =<< genkey (fastMigrate oldbackend) where go Nothing = stop go (Just (newkey, knowngoodcontent)) - | knowngoodcontent = finish (removesize newkey) + | knowngoodcontent = finish =<< tweaksize newkey | otherwise = stopUnless checkcontent $ - finish (removesize newkey) + finish =<< tweaksize newkey checkcontent = Command.Fsck.checkBackend oldbackend oldkey KeyPresent afile finish newkey = ifM (Command.ReKey.linkKey file oldkey newkey) ( do @@ -142,7 +139,7 @@ perform onlyremovesize o file oldkey oldkeyrec oldbackend newbackend = go =<< ge logMigration oldkeyrec , giveup "failed creating link from old to new key" ) - genkey _ | onlyremovesize = return $ Just (oldkey, False) + genkey _ | onlytweaksize = return $ Just (oldkey, False) genkey Nothing = do content <- calcRepo $ gitAnnexLocation oldkey let source = KeySource @@ -155,9 +152,17 @@ perform onlyremovesize o file oldkey oldkeyrec oldbackend newbackend = go =<< ge genkey (Just fm) = fm oldkey newbackend afile >>= \case Just newkey -> return (Just (newkey, True)) Nothing -> genkey Nothing - removesize k - | removeSize o = alterKey k $ \kd -> kd { keySize = Nothing } - | otherwise = k + tweaksize k + | removeSize o = pure (removesize k) + | onlytweaksize = addsize k + | otherwise = pure k + removesize k = alterKey k $ \kd -> kd { keySize = Nothing } + addsize k + | fromKey keySize k == Nothing = + contentSize k >>= return . \case + Just sz -> alterKey k $ \kd -> kd { keySize = Just sz } + Nothing -> k + | otherwise = return k afile = AssociatedFile (Just file) update :: Key -> Key -> CommandStart diff --git a/doc/git-annex-migrate.mdwn b/doc/git-annex-migrate.mdwn index e5e4fe0bec..235a32c5e3 100644 --- a/doc/git-annex-migrate.mdwn +++ b/doc/git-annex-migrate.mdwn @@ -82,6 +82,10 @@ format. git-annex migrate --remove-size --backend=URL somefile + To add back the size to an URL key, use this: + + git-annex migrate --backend=URL somefile + * `--json` Enable JSON output. This is intended to be parsed by programs that use