Merge branch 'checkout'

.gitignore

@@ -1,4 +1,5 @@
 build/*
+test
 git-annex
 git-annex.1
 doc/.ikiwiki

CmdLine.hs

@@ -30,10 +30,13 @@ import qualified Command.SetKey
 import qualified Command.Fix
 import qualified Command.Init
 import qualified Command.Fsck
+import qualified Command.Unlock
+import qualified Command.Lock
+import qualified Command.PreCommit
 
 subCmds :: [SubCommand]
 subCmds =
-	[ SubCommand "add" path	(withFilesNotInGit Command.Add.start)
+	[ SubCommand "add" path	(withFilesToAdd Command.Add.start)
 		"add files to annex"
 	, SubCommand "get" path	(withFilesInGit Command.Get.start)
 		"make content of annexed files available"
@@ -41,12 +44,18 @@ subCmds =
 		"indicate content of files not currently wanted"
 	, SubCommand "move" path	(withFilesInGit Command.Move.start)
 		"transfer content of files to/from another repository"
+	, SubCommand "unlock" path	(withFilesInGit Command.Unlock.start)
+		"unlock files for modification"
+	, SubCommand "edit" path	(withFilesInGit Command.Unlock.start)
+		"same as unlock"
+	, SubCommand "lock" path	(withFilesInGit Command.Lock.start)
+		"undo unlock command"
 	, SubCommand "init" desc	(withDescription Command.Init.start)
 		"initialize git-annex with repository description"
 	, SubCommand "unannex" path	(withFilesInGit Command.Unannex.start)
 		"undo accidential add command"
-	, SubCommand "pre-commit" path (withFilesToBeCommitted Command.Fix.start)
+	, SubCommand "pre-commit" path (withFilesToBeCommitted Command.PreCommit.start)
-		"fix up symlinks before they are committed"
+		"run by git pre-commit hook"
 	, SubCommand "fromkey" key	(withFilesMissing Command.FromKey.start)
 		"adds a file using a specific key"
 	, SubCommand "dropkey"	key	(withKeys Command.DropKey.start)
@@ -106,13 +115,6 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs
 
 {- These functions find appropriate files or other things based on a
    user's parameters. -}
-withFilesNotInGit :: SubCmdSeekBackendFiles
-withFilesNotInGit a params = do
-	repo <- Annex.gitRepo
-	files <- liftIO $ mapM (Git.notInRepo repo) params
-	let files' = foldl (++) [] files
-	pairs <- Backend.chooseBackends files'
-	return $ map a $ filter (\(f,_) -> notState f) pairs
 withFilesInGit :: SubCmdSeekStrings
 withFilesInGit a params = do
 	repo <- Annex.gitRepo
@@ -126,6 +128,14 @@ withFilesMissing a params = do
 		missing f = do
 			e <- doesFileExist f
 			return $ not e
+withFilesToAdd :: SubCmdSeekBackendFiles
+withFilesToAdd a params = do
+	repo <- Annex.gitRepo
+	newfiles <- liftIO $ mapM (Git.notInRepo repo) params
+	unlockedfiles <- liftIO $ mapM (Git.typeChangedFiles repo) params
+	let files = foldl (++) [] $ newfiles ++ unlockedfiles
+	pairs <- Backend.chooseBackends files
+	return $ map a $ filter (\(f,_) -> notState f) pairs
 withDescription :: SubCmdSeekStrings
 withDescription a params = return [a $ unwords params]
 withFilesToBeCommitted :: SubCmdSeekStrings

Command/Lock.hs

@@ -0,0 +1,50 @@
+{- git-annex command
+ -
+ - Copyright 2010 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.Lock where
+
+import Control.Monad.State (liftIO)
+import System.Directory
+import System.Posix.Files
+
+import Types
+import Command
+import Messages
+import qualified Annex
+import qualified GitRepo as Git
+
+{- Undo unlock -}
+start :: SubCmdStartString
+start file = do
+	locked <- isLocked file
+	if locked
+		then return Nothing
+		else do
+			showStart "lock" file
+			return $ Just $ perform file
+
+perform :: FilePath -> SubCmdPerform
+perform file = do
+	liftIO $ removeFile file
+	g <- Annex.gitRepo
+	-- first reset the file to drop any changes checked into the index
+	liftIO $ Git.run g ["reset", "-q", "--", file]
+	-- checkout the symlink
+	liftIO $ Git.run g ["checkout", "--", file]
+	return $ Just $ return True -- no cleanup needed
+
+{- Checks if a file is unlocked for edit.
+ -
+ - But, without the symlink to the annex, cannot tell for sure if the
+ - file was annexed before. So, check if git thinks the file's type has
+ - changed (from a symlink to a regular file). -}
+isLocked :: FilePath -> Annex Bool
+isLocked file = do
+	g <- Annex.gitRepo
+	typechanged <- liftIO $ Git.typeChangedFiles g file
+	s <- liftIO $ getSymbolicLinkStatus file
+	return $ (not $ elem file typechanged) || isSymbolicLink s

Command/PreCommit.hs

@@ -0,0 +1,40 @@
+{- git-annex command
+ -
+ - Copyright 2010 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.PreCommit where
+
+import Control.Monad.State (liftIO)
+import Control.Monad (when, unless)
+
+import Command
+import qualified Annex
+import qualified Backend
+import qualified GitRepo as Git
+import qualified Command.Fix
+import qualified Command.Lock
+import qualified Command.Add
+
+{- Run by git pre-commit hook. -}
+start :: SubCmdStartString
+start file = do
+	-- If a file is unlocked for edit, add its new content to the
+	-- annex, -}
+	locked <- Command.Lock.isLocked file
+	when (not locked) $ do
+		pairs <- Backend.chooseBackends [file]
+		ok <- doSubCmd $ Command.Add.start $ pairs !! 0
+		unless (ok) $ do
+			error $ "failed to add " ++ file ++ "; canceling commit"
+		-- git commit will have staged the file's content;
+		-- drop that and stage the symlink
+		g <- Annex.gitRepo
+		liftIO $ Git.run g ["reset", "-q", "--", file]
+		Annex.queueRun
+
+	-- Fix symlinks as they are committed, this ensures the
+	-- relative links are not broken when moved around.
+	Command.Fix.start file

Command/Unlock.hs

@@ -0,0 +1,39 @@
+{- git-annex command
+ -
+ - Copyright 2010 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.Unlock where
+
+import Control.Monad.State (liftIO)
+import System.Directory
+
+import Command
+import qualified Annex
+import Types
+import Messages
+import Locations
+import Utility
+import Core
+
+{- The unlock subcommand replaces the symlink with a copy of the file's
+ - content. -}
+start :: SubCmdStartString
+start file = isAnnexed file $ \(key, _) -> do
+	showStart "unlock" file
+	return $ Just $ perform file key
+
+perform :: FilePath -> Key -> SubCmdPerform
+perform dest key = do
+	g <- Annex.gitRepo
+	let src = annexLocation g key
+	liftIO $ removeFile dest
+	showNote "copying..."
+	ok <- liftIO $ boolSystem "cp" ["-p", src, dest]
+        if ok
+                then do
+			liftIO $ allowWrite dest
+			return $ Just $ return True
+                else error "cp failed!"

Core.hs

@@ -173,6 +173,7 @@ moveAnnex key src = do
 	let dir = parentDir dest
 	liftIO $ do
 		createDirectoryIfMissing True dir
+		allowWrite dir -- in case the directory already exists
 		renameFile src dest
 		preventWrite dest
 		preventWrite dir

GitRepo.hs

@@ -39,6 +39,7 @@ module GitRepo (
 	checkAttr,
 	decodeGitFile,
 	encodeGitFile,
+	typeChangedFiles,
 
 	prop_idempotent_deencode
 ) where
@@ -58,6 +59,7 @@ import Data.Char
 import Data.Word (Word8)
 import Codec.Binary.UTF8.String (encode)
 import Text.Printf
+import Data.List
 
 import Utility
 
@@ -227,20 +229,31 @@ hPipeRead repo params = assertLocal repo $ do
  - are checked into git at that location. -}
 inRepo :: Repo -> FilePath -> IO [FilePath]
 inRepo repo l = pipeNullSplit repo
-	["ls-files", "--cached", "--exclude-standard", "-z", l]
+	["ls-files", "--cached", "--exclude-standard", "-z", "--", l]
 
 {- Passed a location, recursively scans for all files that are not checked
  - into git, and not gitignored. -}
 notInRepo :: Repo -> FilePath -> IO [FilePath]
 notInRepo repo l = pipeNullSplit repo
-	["ls-files", "--others", "--exclude-standard", "-z", l]
+	["ls-files", "--others", "--exclude-standard", "-z", "--", l]
 
 {- Passed a location, returns a list of the files, staged for
  - commit, that are being added, moved, or changed (but not deleted). -}
 stagedFiles :: Repo -> FilePath -> IO [FilePath]
 stagedFiles repo l = pipeNullSplit repo
-	["diff", "--cached", "--name-only", "--diff-filter=ACMRT", "-z",
+	["diff", "--cached", "--name-only", "--diff-filter=ACMRT", "-z", 
-	 "HEAD", l]
+		"--", l]
+
+{- Passed a location, returns a list of the files whose type has changed. -}
+typeChangedFiles :: Repo -> FilePath -> IO [FilePath]
+typeChangedFiles repo l = do
+	changed <- pipeNullSplit repo $ start ++ end
+	changedCached <- pipeNullSplit repo $ start ++ ["--cached"] ++ end
+	-- a file can be found twice by the above, so nub
+	return $ nub $ changed ++ changedCached
+	where
+		start = ["diff", "--name-only", "--diff-filter=T", "-z"]
+		end = ["--", l]
 
 {- Reads null terminated output of a git command (as enabled by the -z 
  - parameter), and splits it into a list of files. -}

Makefile

@@ -1,8 +1,10 @@
 all: git-annex docs
 
+ghcmake=ghc -Wall -odir build -hidir build --make 
+
 git-annex:
 	mkdir -p build
-	ghc -Wall -odir build -hidir build --make git-annex
+	$(ghcmake) git-annex
 
 install:
 	install -d $(DESTDIR)/usr/bin
@@ -17,7 +19,8 @@ IKIWIKI=ikiwiki
 endif
 
 test:
-	runghc test.hs
+	$(ghcmake) test
+	./test
 
 docs:
 	./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1
@@ -27,7 +30,7 @@ docs:
 		--disable-plugin=smiley
 
 clean:
-	rm -rf build git-annex git-annex.1
+	rm -rf build git-annex git-annex.1 test
 	rm -rf doc/.ikiwiki html
 
-.PHONY: git-annex
+.PHONY: git-annex test

debian/changelog

@@ -1,5 +1,12 @@
 git-annex (0.04) UNRELEASED; urgency=low
 
+  * Add unlock subcommand, which replaces the symlink with a copy of
+    the file's content in preparation of changing it. The "edit" subcommand
+    is an alias for unlock.
+  * Add lock subcommand.
+  * Unlocked files will now automatically be added back into the annex when
+    committed (and the updated symlink committed), by some magic in the
+    pre-commit hook.
   * Add build dep on libghc6-testpack-dev.
   * Add annex.version, which will be used to automate upgrades
     between incompatable versions.
@@ -11,6 +18,9 @@ git-annex (0.04) UNRELEASED; urgency=low
   * Annexed file contents are now made unwritable and put in unwriteable
     directories, to avoid them accidentially being removed or modified.
     (Thanks Josh Triplett for the idea.)
+  * Avoid using runghc to run test suite as it is not available on all
+    architectures. Closes: #603006
+  * Missing build dep. Closes: #603016
 
  -- Joey Hess <joeyh@debian.org>  Mon, 08 Nov 2010 12:36:39 -0400
 

doc/backends.mdwn

@@ -17,7 +17,7 @@ can use different backends for different files.
 * `SHA1` -- This backend stores the file's content in
   `.git/annex/objects/`, with a name based on its sha1 checksum. This backend
   allows modifications of files to be tracked. Its need to generate checksums
-  can make it slower for large files. **Warning** this backend is not ready
+  can make it slower for large files.
   for use.
 * `URL` -- This backend downloads the file's content from an external URL.
 

doc/git-annex.mdwn

@@ -81,6 +81,19 @@ Many git-annex subcommands will stage changes for later `git commit` by you.
   git-annex may refuse to drop content if the backend does not think
   it is safe to do so, typically because of the setting of annex.numcopies.
 
+* unlock [path ...]
+
+  Normally, the content of annexed files is protected from being changed.
+  Unlocking a annexed file allows it to be modified. This replaces the
+  symlink for each specified file with a copy of the file's content.
+  You can then modify it and `git annex add` (or `git commit`) to inject
+  it back into the annex.
+
+* edit [path ...]
+
+  This is an alias for the unlock subcommand. May be easier to remember,
+  if you think of this as allowing you to edit an annexed file.
+
 * move [path ...]
 
   When used with the --to option, moves the content of annexed files from
@@ -93,7 +106,11 @@ Many git-annex subcommands will stage changes for later `git commit` by you.
 
   Initializes git-annex with a description of the git repository,
   and sets up `.gitattributes` and the pre-commit hook.
-  This is an optional, but recommended step.
+
+* lock [path ...]
+
+  Use this to undo an unlock command if you don't want to modify
+  the files, or have made modifications you want to discard.
 
 * unannex [path ...]
 
@@ -110,7 +127,8 @@ Many git-annex subcommands will stage changes for later `git commit` by you.
 * pre-commit [path ...]
 
   Fixes up symlinks that are staged as part of a commit, to ensure they
-  point to annexed content. 
+  point to annexed content. Also handles injecting changes to unlocked
+  files into the annex.
 
   This is meant to be called from git's pre-commit hook. `git annex init`
   automatically creates a pre-commit hook using this.

doc/todo/backendSHA1.mdwn

@@ -3,3 +3,5 @@ This backend is not finished.
 In particular, while files can be added using it, git-annex will not notice
 when their content changes, and will not create a new key for the new sha1
 of the net content.
+
+[[done]]; use unlock subcommand and commit changes with git

doc/todo/checkout.mdwn

@@ -3,3 +3,21 @@ file's content, with a copy of the file. Once you've checked a file out,
 you can edit it, and `git commit` it. On commit, git-annex will detect
 if the file has been changed, and if it has, `add` its content to the
 annex.
+
+> Internally, this will need to store the original symlink to the file, in 
+> `.git/annex/checkedout/$filename`.
+> 
+> * git-annex uncheckout moves that back
+> * git-annex pre-commit hook checks each file being committed to see if
+>   it has a symlink there, and if so, removes the symlink and adds the new
+>   content to the annex. 
+> 
+> And it seems the file content should be copied, not moved or hard linked:
+> 
+> * Makes sure other annexes can find it if transferring it from
+>   this annex.
+> * Ensures it's always available for uncheckout.
+> * Avoids the last copy of a file's content being lost when
+>   the checked out file is modified.
+
+[[done]]

doc/walkthrough.mdwn

@@ -139,6 +139,50 @@ But `other.iso` looks to have never been copied to anywhere else, so if
 it's something you want to hold onto, you'd need to transfer it to
 some other repository before dropping it.
 
+## modifying annexed files
+
+Normally, the content of files in the annex is prevented from being modified.
+
+	# echo oops > my_cool_big_file
+	bash: my_cool_big_file: Permission deined
+
+In order to modify a file, it should first be unlocked.
+
+	# git annex unlock my_cool_big_file
+	unlock my_cool_big_file (copying...) ok
+
+They replaces the symlink that normally points at its content with a copy
+of the content. You can then modify the file like any regular file. Because
+it is a regular file.
+
+If you decide you don't need to modify the file after all, or want to discard
+modifications, just use `git annex lock`.
+
+When you `git commit`, git-annex's pre-commit hook will automatically
+notice that you are committing an unlocked file, and add its new content
+to the annex. The file will be replaced with a symlink to the new content,
+and this symlink is what gets committed to git in the end.
+
+	# echo "now smaller, but even cooler" > my_cool_big_file
+	# git commit my_cool_big_file -m "changed an annexed file"
+	add my_cool_big_file ok
+	[master 64cda67] changed an annexed file
+	 2 files changed, 2 insertions(+), 1 deletions(-)
+	 create mode 100644 .git-annex/SHA1:0b1d8616d0238cb9418a0e0a649bdad2e9e7faae.log
+
+There is one problem with using `git commit` like this: Git wants to first
+stage the entire contents of the file in its index. That can be slow for
+big files (sorta why git-annex exists in the first place). So, the
+automatic handling on commit is a nice safety feature, since it prevents
+the file content being accidentially commited into git. But when working with
+big files, it's faster to explicitly add them to the annex yourself
+before committing.
+
+	# echo "now smaller, but even cooler yet" > my_cool_big_file
+	# git annex add my_cool_big_file
+	add my_cool_big_file ok
+	# git commit my_cool_big_file -m "changed an annexed file"
+
 ## using ssh remotes
 
 So far in this walkthrough, git-annex has been used with a remote
@@ -216,3 +260,17 @@ that the URL is stable; no local backup is kept.
 
 	# git annex drop somefile
 	drop somefile (ok)
+
+## using the SHA1 backend
+
+Another handy alternative to the default [[backend|backends]] is the
+SHA1 backend. This backend provides more git-style assurance that your data
+has not been damanged. And the checksum means that when you add the same
+content to the annex twice, only one copy need be stored in the backend.
+
+The only reason it's not the default is that it needs to checksum
+files when they're added to the annex, and this can slow things down
+significantly for really big files. To make SHA1 the detault, just
+add something like this to `.gitattributes`:
+
+	* git-annex-backend=SHA1

test.hs

@@ -1,15 +1,16 @@
 -- TODO find a test harness that is actually in Debian and use it.
 
-import Test.QuickCheck
 import Test.HUnit
 import Test.HUnit.Tools
 
 import GitRepo
 import Locations
 
+alltests :: [Test]
 alltests = [
 	qctest "prop_idempotent_deencode" prop_idempotent_deencode,
 	qctest "prop_idempotent_fileKey" prop_idempotent_fileKey
 	]
 
+main :: IO (Counts, Int)
 main = runVerboseTests (TestList alltests)