migrate: Support migrating v1 SHA keys to v2 SHA keys with size information that can be used for free space checking.

Backend.hs

@@ -24,6 +24,7 @@ module Backend (
 	removeKey,
 	hasKey,
 	fsckKey,
+	upgradableKey,
 	lookupFile,
 	chooseBackends,
 	keyBackend,
@@ -130,6 +131,10 @@ fsckKey backend key file numcopies = do
 	backend_ok <-(B.fsckKey backend) key file numcopies
 	return $ size_ok && backend_ok
 
+{- Checks if a key is upgradable to a newer representation. -}
+upgradableKey :: Backend Annex -> Key -> Annex Bool
+upgradableKey backend key = (B.upgradableKey backend) key
+
 {- Looks up the key and backend corresponding to an annexed file,
  - by examining what the file symlinks to. -}
 lookupFile :: FilePath -> Annex (Maybe (Key, Backend Annex))

Backend/File.hs

@@ -29,6 +29,7 @@ import Types
 import UUID
 import Messages
 import Trust
+import Key
 
 backend :: Backend Annex
 backend = Backend {
@@ -38,7 +39,8 @@ backend = Backend {
 	retrieveKeyFile = copyKeyFile,
 	removeKey = checkRemoveKey,
 	hasKey = inAnnex,
-	fsckKey = checkKeyOnly
+	fsckKey = checkKeyOnly,
+	upgradableKey = checkUpgradableKey
 }
 
 mustProvide :: a
@@ -159,6 +161,12 @@ getNumCopies Nothing = do
 	where
 		config = "annex.numcopies"
 
+{- Ideally, all keys have file size metadata. Old keys may not. -}
+checkUpgradableKey :: Key -> Annex Bool
+checkUpgradableKey key
+	| keySize key == Nothing = return True
+	| otherwise = return False
+
 {- This is used to check that numcopies is satisfied for the key on fsck.
  - This trusts data in the the location log, and so can check all keys, even
  - those with data not present in the current annex.

Backend/URL.hs

@@ -30,7 +30,9 @@ backend = Backend {
 	-- similarly, keys are always assumed to be out there on the web
 	hasKey = dummyOk,
 	-- and nothing needed to fsck
-	fsckKey = dummyFsck
+	fsckKey = dummyFsck,
+	-- and key upgrade not needed
+	upgradableKey = \_ -> return False
 }
 
 -- cannot generate url from filename

BackendClass.hs

@@ -29,7 +29,9 @@ data Backend a = Backend {
 	-- (second parameter may be the filename associated with it)
 	-- (third parameter may be the number of copies that there should
 	-- be of the key)
-	fsckKey :: Key -> Maybe FilePath -> Maybe Int -> a Bool
+	fsckKey :: Key -> Maybe FilePath -> Maybe Int -> a Bool,
+	-- Is a newer repesentation possible for a key?
+	upgradableKey :: Key -> a Bool
 }
 
 instance Show (Backend a) where

Command/Migrate.hs

@@ -31,8 +31,8 @@ start :: CommandStartBackendFile
 start (file, b) = isAnnexed file $ \(key, oldbackend) -> do
 	exists <- inAnnex key
 	newbackend <- choosebackend b
-	force <- Annex.getState Annex.force
-	if (newbackend /= oldbackend || force) && exists
+	upgradable <- Backend.upgradableKey oldbackend key
+	if (newbackend /= oldbackend || upgradable) && exists
 		then do
 			showStart "migrate" file
 			return $ Just $ perform file key newbackend

debian/changelog

@@ -12,6 +12,8 @@ git-annex (0.20110321) UNRELEASED; urgency=low
   * fsck: In fast mode, avoid checking checksums.
   * unused: In fast mode, just show all existing temp files as unused,
     and avoid expensive scan for other unused content.
+  * migrate: Support migrating v1 SHA keys to v2 SHA keys with
+    size information that can be used for free space checking.
   * Fix space leak in fsck and drop commands.
   * migrate: Bugfix for case when migrating a file results in a key that
     is already present in .git/annex/objects.

doc/git-annex.mdwn

@@ -162,10 +162,15 @@ Many git-annex commands will stage changes for later `git commit` by you.
 * migrate [path ...]
 
   Changes the specified annexed files to store their content in the
-  default backend (or the one specified with --backend).
+  default backend (or the one specified with --backend). Only files whose
+  content is currently available are migrated.
 
   Note that the content is not removed from the backend it was previously in.
-  Use `git annex unused` to find and remove such content. 
+  Use `git annex unused` to find and remove such content.
+
+  Normally, nothing will be done to files already in the backend.
+  However, if a backend changes the information it uses to construct a key,
+  this can also be used to migrate files to use the new key format.
 
 * map
 

doc/upgrades.mdwn

@@ -56,9 +56,9 @@ And .gitattributes needed to have another line added to it.
 Previously, files added to the SHA [[backends]] did not have their file
 size tracked, while files added to the WORM backend did. Files added to
 the SHA backends after the conversion will have their file size tracked,
-and that information will be used by git-annex for disk space checking.
-There is not yet a way to add file size tracking information to old files
-in the SHA backend.
+and that information will be used by git-annex for disk free space checking.
+To ensure that information is available for all your annexed files, see
+[[upgrades/SHA_size]].
 
 ### v0 -> v1 (git-annex version 0.03 to version 0.04)
 

doc/upgrades/SHA_size.mdwn

@@ -0,0 +1,20 @@
+Before version 2 of the git-annex repository, files added to the SHA
+[[backends]] did not have their file size tracked, while files added to the
+WORM backend did. The file size information is used for disk free space
+checking.
+
+Files added to the SHA backends after the conversion will have their file
+size tracked automatically. This disk free space checking is an optional
+feature and since you're more likely to be using more recently added files,
+you're unlikely to see any bad effect if you do nothing.
+
+That said, if you have old files added to SHA backends that lack file size
+tracking info, here's how you can add that info. After [[upgrading|upgrades]]
+to repository version 2, in each repository run:
+
+	git annex migrate
+	git commit -m 'migrated keys for v2'
+
+The usual caveats about [[walkthrough/migrating_data_to_a_new_backend]]
+apply; you will end up with unused keys that you can later clean up with
+`git annex unused`.