migrate: support adding size back to URL keys

migrate: Support adding size to URL keys that were added with --relaxed, by
running eg: git-annex migrate --backend=URL foo

Since url keys cannot be generated, that used to fail. Make it notice that
the backend is not changed, and just get the size of the content.

Sponsored-by: Brock Spratlen on Patreon

Annex/Content.hs

@@ -1,6 +1,6 @@
 {- git-annex file content managing
  -
- - Copyright 2010-2022 Joey Hess <id@joeyh.name>
+ - Copyright 2010-2023 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
@@ -66,6 +66,7 @@ module Annex.Content (
 	getKeyStatus,
 	getKeyFileStatus,
 	cleanObjectDirs,
+	contentSize,
 ) where
 
 import System.IO.Unsafe (unsafeInterleaveIO)
@@ -916,3 +917,25 @@ getKeyFileStatus key file = do
 				)
 		_ -> return s
 
+{- Gets the size of the content of a key when it is present.
+ - Useful when the key does not have keySize set. 
+ -
+ - When the object file appears possibly modified with annex.thin set, does
+ - not do an expensive verification that the content is good, just returns
+ - Nothing.
+ -}
+contentSize :: Key -> Annex (Maybe FileSize)
+contentSize key = catchDefaultIO Nothing $
+	withObjectLoc key $ \loc ->
+		withTSDelta (liftIO . genInodeCache loc) >>= \case
+			Just ic -> ifM (unmodified ic)
+				( return (Just (inodeCacheFileSize ic))
+				, return Nothing
+				)
+			Nothing -> return Nothing
+  where
+	unmodified ic =
+		ifM (annexThin <$> Annex.getGitConfig)
+			( isUnmodifiedCheap' key ic
+			, return True
+			)

Annex/Content/Presence.hs

@@ -18,6 +18,7 @@ module Annex.Content.Presence (
 	isUnmodified,
 	isUnmodified',
 	isUnmodifiedCheap,
+	isUnmodifiedCheap',
 	withContentLockFile,
 	contentLockFile,
 ) where
@@ -206,7 +207,7 @@ isUnmodified' = isUnmodifiedLowLevel Database.Keys.addInodeCaches
  - within a small time window (eg 1 second).
  -}
 isUnmodifiedCheap :: Key -> RawFilePath -> Annex Bool
-isUnmodifiedCheap key f = maybe (return False) (isUnmodifiedCheap' key) 
+isUnmodifiedCheap key f = maybe (pure False) (isUnmodifiedCheap' key) 
 	=<< withTSDelta (liftIO . genInodeCache f)
 
 isUnmodifiedCheap' :: Key -> InodeCache -> Annex Bool

CHANGELOG

@@ -9,6 +9,8 @@ git-annex (10.20231130) UNRELEASED; urgency=medium
     that have been migrated.
   * Added annex.syncmigrations config that can be set to false to prevent
     pull and sync from migrating object content.
+  * migrate: Support adding size to URL keys that were added with
+    --relaxed, by running eg: git-annex migrate --backend=URL foo
   * Make git-annex get/copy/move --from foo override configuration of 
     remote.foo.annex-ignore, as documented.
   * Support git-annex copy/move --from-anywhere --to remote.

Command/Migrate.hs

@@ -88,29 +88,26 @@ start o ksha si file key = do
 		Just oldbackend -> do
 			exists <- inAnnex key
 			newbackend <- chooseBackend file
-			if (newbackend /= oldbackend || upgradableKey oldbackend key || forced) && exists
+			if (newbackend /= oldbackend || upgradableKey oldbackend || forced) && exists
 				then go False oldbackend newbackend
-				else if removeSize o && exists
+				else if cantweaksize newbackend oldbackend && exists
-					then go True oldbackend oldbackend
+					then go True oldbackend newbackend
 					else stop
   where
-	go onlyremovesize oldbackend newbackend = do
+	go onlytweaksize oldbackend newbackend = do
 		keyrec <- case ksha of
 			Just (KeySha s) -> pure (MigrationRecord s)
 			Nothing -> error "internal"
 		starting "migrate" (mkActionItem (key, file)) si $
-			perform onlyremovesize o file key keyrec oldbackend newbackend
+			perform onlytweaksize o file key keyrec oldbackend newbackend
 
-{- Checks if a key is upgradable to a newer representation.
+	cantweaksize newbackend oldbackend
- - 
+		| removeSize o = isJust (fromKey keySize key)
- - Reasons for migration:
+		| newbackend /= oldbackend = False
- -  - Ideally, all keys have file size metadata. Old keys may not.
+		| isNothing (fromKey keySize key) = True
- -  - Something has changed in the backend, such as a bug fix.
+		| otherwise = False
- -}
+
-upgradableKey :: Backend -> Key -> Bool
+	upgradableKey oldbackend = maybe False (\a -> a key) (canUpgradeKey oldbackend)
-upgradableKey backend key = isNothing (fromKey keySize key) || backendupgradable
-  where
-	backendupgradable = maybe False (\a -> a key) (canUpgradeKey backend)
 
 {- Store the old backend's key in the new backend
  - The old backend's key is not dropped from it, because there may
@@ -122,13 +119,13 @@ upgradableKey backend key = isNothing (fromKey keySize key) || backendupgradable
  - generated.
  -}
 perform :: Bool -> MigrateOptions -> RawFilePath -> Key -> MigrationRecord -> Backend -> Backend -> CommandPerform
-perform onlyremovesize o file oldkey oldkeyrec oldbackend newbackend = go =<< genkey (fastMigrate oldbackend)
+perform onlytweaksize o file oldkey oldkeyrec oldbackend newbackend = go =<< genkey (fastMigrate oldbackend)
   where
 	go Nothing = stop
 	go (Just (newkey, knowngoodcontent))
-		| knowngoodcontent = finish (removesize newkey)
+		| knowngoodcontent = finish =<< tweaksize newkey
 		| otherwise = stopUnless checkcontent $
-			finish (removesize newkey)
+			finish =<< tweaksize newkey
 	checkcontent = Command.Fsck.checkBackend oldbackend oldkey KeyPresent afile
 	finish newkey = ifM (Command.ReKey.linkKey file oldkey newkey)
 		( do
@@ -142,7 +139,7 @@ perform onlyremovesize o file oldkey oldkeyrec oldbackend newbackend = go =<< ge
 				logMigration oldkeyrec
 		, giveup "failed creating link from old to new key"
 		)
-	genkey _ | onlyremovesize = return $ Just (oldkey, False)
+	genkey _ | onlytweaksize = return $ Just (oldkey, False)
 	genkey Nothing = do
 		content <- calcRepo $ gitAnnexLocation oldkey
 		let source = KeySource
@@ -155,9 +152,17 @@ perform onlyremovesize o file oldkey oldkeyrec oldbackend newbackend = go =<< ge
 	genkey (Just fm) = fm oldkey newbackend afile >>= \case
 		Just newkey -> return (Just (newkey, True))
 		Nothing -> genkey Nothing
-	removesize k
+	tweaksize k
-		| removeSize o = alterKey k $ \kd -> kd { keySize = Nothing } 
+		| removeSize o = pure (removesize k)
-		| otherwise = k
+		| onlytweaksize = addsize k
+		| otherwise = pure k
+	removesize k = alterKey k $ \kd -> kd { keySize = Nothing }
+	addsize k
+		| fromKey keySize k == Nothing = 
+			contentSize k >>= return . \case
+				Just sz -> alterKey k $ \kd -> kd { keySize = Just sz }
+				Nothing -> k
+		| otherwise = return k
 	afile = AssociatedFile (Just file)
 
 update :: Key -> Key -> CommandStart

doc/git-annex-migrate.mdwn

@@ -82,6 +82,10 @@ format.
 
   	git-annex migrate --remove-size --backend=URL somefile
 
+  To add back the size to an URL key, use this:
+
+    git-annex migrate --backend=URL somefile
+
 * `--json`
 
   Enable JSON output. This is intended to be parsed by programs that use