reuse hashes of dotfiles/dirs/submodules when entering view

This fixes a crash when a git submodule has a name starting with a dot.
Such a submodule might contain dotfiles that are intended to be used when
inside the view (since a dot-directory that's not a submodule was already
preserved when entering a view). So, rather than eliminating the submodule
from the view, its git ls-files --stage hash is copied over into the view.

dotfiles/dirs have their git ls-files --stage hashes similarly copied over
to the view. This is more efficient and simpler than the old method,
and also won't break if git ever adds a new type of tree item, like was
done with submodules.

Since the content of dotfiles in the working tree is no longer hashed
when entering a view, when there are unstaged modifications, they are
not included in the view branch. Entering the view branch still works,
but git checkout shows "M .dotfile", and git diff will show the unstaged
changes. This seems like an improvement over the old behavior.

Also made Command.View not delete empty directories that are submodules
when entering a view, while still deleting other empty directories.

This commit was supported by the NSF-funded DataLad project.

Annex/View.hs

@@ -19,7 +19,6 @@ import qualified Git.LsFiles
 import qualified Git.Ref
 import Git.UpdateIndex
 import Git.Sha
-import Annex.HashObject
 import Git.Types
 import Git.FilePath
 import Annex.WorkTree
@@ -29,7 +28,6 @@ import Annex.CatFile
 import Logs.MetaData
 import Logs.View
 import Utility.Glob
-import Utility.FileMode
 import Types.Command
 import CmdLine.Action
 
@@ -327,8 +325,9 @@ applyView = applyView' viewedFileFromReference getWorkTreeMetaData
 narrowView :: View -> Annex Git.Branch
 narrowView = applyView' viewedFileReuse getViewedFileMetaData
 
-{- Go through each file in the currently checked out branch.
+{- Go through each staged file.
- - If the file is not annexed, skip it, unless it's a dotfile in the top.
+ - If the file is not annexed, skip it, unless it's a dotfile in the top,
+ - or a file in a dotdir in the top. 
  - Look up the metadata of annexed files, and generate any ViewedFiles,
  - and stage them.
  -
@@ -337,39 +336,32 @@ narrowView = applyView' viewedFileReuse getViewedFileMetaData
 applyView' :: MkViewedFile -> (FilePath -> MetaData) -> View -> Annex Git.Branch
 applyView' mkviewedfile getfilemetadata view = do
 	top <- fromRepo Git.repoPath
-	(l, clean) <- inRepo $ Git.LsFiles.inRepo [top]
+	(l, clean) <- inRepo $ Git.LsFiles.stagedDetails [top]
 	liftIO . nukeFile =<< fromRepo gitAnnexViewIndex
 	uh <- withViewIndex $ inRepo Git.UpdateIndex.startUpdateIndex
-	forM_ l $ \f -> do
+	forM_ l $ \(f, sha, mode) -> do
 		topf <- inRepo (toTopFilePath f)
-		go uh topf =<< lookupFile f
+		go uh topf sha (toTreeItemType =<< mode) =<< lookupFile f
 	liftIO $ do
 		void $ stopUpdateIndex uh
 		void clean
 	genViewBranch view
   where
 	genviewedfiles = viewedFiles view mkviewedfile -- enables memoization
-	go uh topf (Just k) = do
+
+	go uh topf _sha _mode (Just k) = do
 		metadata <- getCurrentMetaData k
 		let f = getTopFilePath topf
 		let metadata' = getfilemetadata f `unionMetaData` metadata
 		forM_ (genviewedfiles f metadata') $ \fv -> do
 			f' <- fromRepo $ fromTopFilePath $ asTopFilePath fv
 			stagesymlink uh f' =<< calcRepo (gitAnnexLink f' k)
-	go uh topf Nothing
+	go uh topf (Just sha) (Just treeitemtype) Nothing
-		| "." `isPrefixOf` getTopFilePath topf = do
+		| "." `isPrefixOf` getTopFilePath topf =
-			f <- fromRepo $ fromTopFilePath topf
+			liftIO $ Git.UpdateIndex.streamUpdateIndex' uh $
-			s <- liftIO $ getSymbolicLinkStatus f
+				pureStreamer $ updateIndexLine sha treeitemtype topf
-			if isSymbolicLink s
+	go _ _ _ _  _ = noop
-				then stagesymlink uh f =<< liftIO (readSymbolicLink f)
+
-				else do
-					sha <- hashFile f
-					let blobtype = if isExecutable (fileMode s)
-						then ExecutableBlob
-						else FileBlob
-					liftIO . Git.UpdateIndex.streamUpdateIndex' uh
-						=<< inRepo (Git.UpdateIndex.stageFile sha blobtype f)
-		| otherwise = noop
 	stagesymlink uh f linktarget = do
 		sha <- hashSymlink linktarget
 		liftIO . Git.UpdateIndex.streamUpdateIndex' uh

CHANGELOG

@@ -1,3 +1,11 @@
+git-annex (6.20180510) UNRELEASED; urgency=medium
+
+  * view, vadd: Fix crash when a git submodule has a name starting with a dot.
+  * Improve handling of unstaged modifications to dotfiles when entering a
+    view.
+
+ -- Joey Hess <id@joeyh.name>  Mon, 14 May 2018 13:42:41 -0400
+
 git-annex (6.20180509) upstream; urgency=medium
 
   * The old git-annex Android app is now deprecated in favor of running

Command/View.hs

@@ -12,6 +12,8 @@ import qualified Git
 import qualified Git.Command
 import qualified Git.Ref
 import qualified Git.Branch
+import qualified Git.LsFiles as LsFiles
+import Git.FilePath
 import Types.View
 import Annex.View
 import Logs.View
@@ -65,15 +67,21 @@ checkoutViewBranch view mkbranch = do
 	when ok $ do
 		setView view
 		{- A git repo can easily have empty directories in it,
-		 - and this pollutes the view, so remove them. -}
+		 - and this pollutes the view, so remove them.
-		top <- fromRepo Git.repoPath
+		 - (However, emptry directories used by submodules are not
-		liftIO $ removeemptydirs top
+		 - removed.) -}
+		top <- liftIO . absPath =<< fromRepo Git.repoPath
+		(l, cleanup) <- inRepo $
+			LsFiles.notInRepoIncludingEmptyDirectories False [top]
+		forM_ l (removeemptydir top)
+		liftIO $ void cleanup
 		unlessM (liftIO $ doesDirectoryExist here) $ do
 			showLongNote (cwdmissing top)
 	return ok
   where
-	removeemptydirs top = mapM_ (tryIO . removeDirectory)
+	removeemptydir top d = do
-		=<< dirTreeRecursiveSkipping (".git" `isSuffixOf`) top
+		p <- inRepo $ toTopFilePath d
+		liftIO $ tryIO $ removeDirectory (top </> getTopFilePath p)
 	cwdmissing top = unlines
 		[ "This view does not include the subdirectory you are currently in."
 		, "Perhaps you should:  cd " ++ top

Git/LsFiles.hs

@@ -1,6 +1,6 @@
 {- git ls-files interface
  -
- - Copyright 2010,2012 Joey Hess <id@joeyh.name>
+ - Copyright 2010-2018 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -8,6 +8,7 @@
 module Git.LsFiles (
 	inRepo,
 	notInRepo,
+	notInRepoIncludingEmptyDirectories,
 	allFiles,
 	deleted,
 	modified,
@@ -44,10 +45,14 @@ inRepo l = pipeNullSplit $
 
 {- Scans for files at the specified locations that are not checked into git. -}
 notInRepo :: Bool -> [FilePath] -> Repo -> IO ([FilePath], IO Bool)
-notInRepo include_ignored l repo = pipeNullSplit params repo
+notInRepo = notInRepo' []
+
+notInRepo' :: [CommandParam] -> Bool -> [FilePath] -> Repo -> IO ([FilePath], IO Bool)
+notInRepo' ps include_ignored l repo = pipeNullSplit params repo
   where
 	params = concat
 		[ [ Param "ls-files", Param "--others"]
+		, ps
 		, exclude
 		, [ Param "-z", Param "--" ]
 		, map File l
@@ -56,6 +61,11 @@ notInRepo include_ignored l repo = pipeNullSplit params repo
 		| include_ignored = []
 		| otherwise = [Param "--exclude-standard"]
 
+{- Scans for files at the specified locations that are not checked into
+ - git. Empty directories are included in the result. -}
+notInRepoIncludingEmptyDirectories :: Bool -> [FilePath] -> Repo -> IO ([FilePath], IO Bool)
+notInRepoIncludingEmptyDirectories = notInRepo' [Param "--directory"]
+
 {- Finds all files in the specified locations, whether checked into git or
  - not. -}
 allFiles :: [FilePath] -> Repo -> IO ([FilePath], IO Bool)

doc/bugs/annex_view_barfs__fatal__58___Unable_to_add___40__null__41___to_database.mdwn

@@ -32,3 +32,4 @@ copy of the repository will be provided via email
 
 
 
+> [[fixed|done]] --[[Joey]]