add: Improved detection of files that are modified while being added.

In indirect mode, now checks the inode cache to detect changes to a file.
Note that a file can still be changed if a process has it open for write,
after landing in the annex.

In direct mode, some checking of the inode cache was done before, but
from a much later point, so fewer modifications could be detected. Now it's
as good as indirect mode.

On crippled filesystems, no lock down is done before starting to add a
file, so checking the inode cache is the only protection we have.

Annex/Direct.hs

@@ -79,6 +79,7 @@ addDirect file cache = do
 	let source = KeySource
 		{ keyFilename = file
 		, contentLocation = file
+		, inodeCache = Just cache
 		}
 	got =<< genKey source =<< chooseBackend file
   where

Command/Add.hs

@@ -28,6 +28,7 @@ import Config
 import qualified Git.HashObject
 import qualified Git.UpdateIndex
 import Git.Types
+import Utility.InodeCache
 
 def :: [Command]
 def = [notBareRepo $ command "add" paramPaths seek "add files to annex"]
@@ -68,8 +69,13 @@ start file = ifAnnexed file fixup add
  -}
 lockDown :: FilePath -> Annex (Maybe KeySource)
 lockDown file = ifM (crippledFileSystem)
-	( return $ Just $
+	( liftIO $ catchMaybeIO $ do
-		KeySource { keyFilename = file, contentLocation = file }
+		cache <- genInodeCache file
+		return $ KeySource
+			{ keyFilename = file
+			, contentLocation = file
+			, inodeCache = cache
+			}
 	, do
 		tmp <- fromRepo gitAnnexTmpDir
 		createAnnexDirectory tmp
@@ -79,7 +85,12 @@ lockDown file = ifM (crippledFileSystem)
 			hClose h
 			nukeFile tmpfile
 			createLink file tmpfile
-			return $ KeySource { keyFilename = file , contentLocation = tmpfile }
+			cache <- genInodeCache tmpfile
+			return $ KeySource
+				{ keyFilename = file
+				, contentLocation = tmpfile
+				, inodeCache = cache
+				}
 	)
 
 {- Ingests a locked down file into the annex.
@@ -91,33 +102,31 @@ ingest :: (Maybe KeySource) -> Annex (Maybe Key)
 ingest Nothing = return Nothing
 ingest (Just source) = do
 	backend <- chooseBackend $ keyFilename source
-	ifM isDirect
+	k <- genKey source backend
-		( do
+	cache <- liftIO $ genInodeCache $ contentLocation source
-			mstat <- liftIO $ catchMaybeIO $ getSymbolicLinkStatus $ keyFilename source
+	case inodeCache source of
-			k <- genKey source backend
+		Nothing -> go k cache
-			godirect k (toInodeCache =<< mstat)
+		Just c
-		, go =<< genKey source backend
+			| (Just c == cache) -> go k cache
-		)
+			| otherwise -> failure
   where
-	go (Just (key, _)) = do
+	go k cache = ifM isDirect ( godirect k cache , goindirect k cache )
+
+	goindirect (Just (key, _)) _ = do
 		handle (undo (keyFilename source) key) $
 			moveAnnex key $ contentLocation source
 		liftIO $ nukeFile $ keyFilename source
 		return $ Just key
-	go Nothing = failure
+	goindirect Nothing _ = failure
 
-	godirect (Just (key, _)) (Just cache) =
+	godirect (Just (key, _)) (Just cache) = do
-		ifM (liftIO $ compareInodeCache (keyFilename source) $ Just cache)
+		writeInodeCache key cache
-			( do
+		void $ addAssociatedFile key $ keyFilename source
-				writeInodeCache key cache
+		unlessM crippledFileSystem $
-				void $ addAssociatedFile key $ keyFilename source
+			liftIO $ allowWrite $ keyFilename source
-				unlessM crippledFileSystem $
+		when (contentLocation source /= keyFilename source) $
-					liftIO $ allowWrite $ keyFilename source
+			liftIO $ nukeFile $ contentLocation source
-				when (contentLocation source /= keyFilename source) $
+		return $ Just key
-					liftIO $ nukeFile $ contentLocation source
-				return $ Just key
-			, failure
-			)
 	godirect _ _ = failure
 
 	failure = do

Command/AddUrl.hs

@@ -75,7 +75,11 @@ download url file = do
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
 	stopUnless (downloadUrl [url] tmp) $ do
 		backend <- chooseBackend file
-		let source = KeySource { keyFilename = file, contentLocation = tmp }
+		let source = KeySource
+			{ keyFilename = file
+			, contentLocation = tmp
+			, inodeCache = Nothing
+			}
 		k <- genKey source backend
 		case k of
 			Nothing -> stop

Command/Migrate.hs

@@ -63,5 +63,9 @@ perform file oldkey oldbackend newbackend = do
 		next $ Command.ReKey.cleanup file oldkey newkey
 	genkey = do
 		content <- inRepo $ gitAnnexLocation oldkey
-		let source = KeySource { keyFilename = file, contentLocation = content }
+		let source = KeySource
+			{ keyFilename = file
+			, contentLocation = content
+			, inodeCache = Nothing
+			}
 		liftM fst <$> genKey source (Just newbackend)

Types/KeySource.hs

@@ -7,6 +7,8 @@
 
 module Types.KeySource where
 
+import Utility.InodeCache
+
 {- When content is in the process of being added to the annex,
  - and a Key generated from it, this data type is used. 
  -
@@ -16,10 +18,12 @@ module Types.KeySource where
  - for checking. The migrate command uses the content
  - of a different Key.
  -
- - 
+ - The inodeCache can be used to detect some types of modifications to
+ - files that may be made while they're in the process of being added.
  -}
 data KeySource = KeySource
 	{ keyFilename :: FilePath
 	, contentLocation :: FilePath
+	, inodeCache :: Maybe InodeCache
 	}
 	deriving (Show)

debian/changelog

@@ -8,6 +8,7 @@ git-annex (3.20130208) UNRELEASED; urgency=low
     support hard links, or symlinks, or unix permissions, and set
     annex.crippledfilesystem, as well as annex.direct. This allows
     use of git-annex repositories on FAT and even worse filesystems.
+  * add: Improved detection of files that are modified while being added.
 
  -- Joey Hess <joeyh@debian.org>  Sun, 10 Feb 2013 14:52:01 -0400