Merge branch 'master' into ghc7.0

Backend.hs

@@ -75,16 +75,16 @@ genKey' (b:bs) file = do
  - by examining what the file symlinks to. -}
 lookupFile :: FilePath -> Annex (Maybe (Key, Backend))
 lookupFile file = do
-	tl <- liftIO $ tryIO getsymlink
+	tl <- liftIO $ tryIO $ readSymbolicLink file
 	case tl of
 		Left _ -> return Nothing
 		Right l -> makekey l
 	where
-		getsymlink = takeFileName <$> readSymbolicLink file
-		makekey l = maybe (return Nothing) (makeret l) (fileKey l)
+		makekey l = maybe (return Nothing) (makeret l) (fileKey $ takeFileName l)
 		makeret l k = let bname = keyBackendName k in
 			case maybeLookupBackendName bname of
-				Just backend -> return $ Just (k, backend)
+				Just backend -> do
+					return $ Just (k, backend)
 				Nothing -> do
 					when (isLinkToAnnex l) $ warning $
 						"skipping " ++ file ++

Command/Import.hs

@@ -0,0 +1,39 @@
+{- git-annex command
+ -
+ - Copyright 2012 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.Import where
+
+import Common.Annex
+import Command
+import qualified Annex
+import qualified Command.Add
+
+def :: [Command]
+def = [command "import" paramPaths seek "move and add files from outside git working copy"]
+
+seek :: [CommandSeek]
+seek = [withPathContents start]
+
+start :: (FilePath, FilePath) -> CommandStart
+start (srcfile, destfile) = notBareRepo $
+	ifM (liftIO $ isRegularFile <$> getSymbolicLinkStatus srcfile)
+		( do
+			showStart "import" destfile
+			next $ perform srcfile destfile
+		, stop
+		)
+
+perform :: FilePath -> FilePath -> CommandPerform
+perform srcfile destfile = do
+	whenM (liftIO $ doesFileExist destfile) $
+		unlessM (Annex.getState Annex.force) $
+			error $ "not overwriting existing " ++ destfile ++
+				" (use --force to override)"
+ 
+	liftIO $ createDirectoryIfMissing True (parentDir destfile)
+	liftIO $ moveFile srcfile destfile
+	Command.Add.perform destfile

Command/Lock.hs

@@ -24,9 +24,5 @@ start file = do
 
 perform :: FilePath -> CommandPerform
 perform file = do
-	liftIO $ removeFile file
-	-- Checkout from HEAD to get rid of any changes that might be 
-	-- staged in the index, and get back to the previous symlink to
-	-- the content.
-	Annex.Queue.add "checkout" [Param "HEAD", Param "--"] [file]
+	Annex.Queue.add "checkout" [Param "--"] [file]
 	next $ return True -- no cleanup needed

GitAnnex.hs

@@ -54,6 +54,7 @@ import qualified Command.Semitrust
 import qualified Command.Dead
 import qualified Command.Sync
 import qualified Command.AddUrl
+import qualified Command.Import
 import qualified Command.Map
 import qualified Command.Upgrade
 import qualified Command.Version
@@ -69,6 +70,7 @@ cmds = concat
 	, Command.Lock.def
 	, Command.Sync.def
 	, Command.AddUrl.def
+	, Command.Import.def
 	, Command.Init.def
 	, Command.Describe.def
 	, Command.InitRemote.def

Locations.hs

@@ -155,7 +155,9 @@ gitAnnexRemotesDir r = addTrailingPathSeparator $ gitAnnexDir r </> "remotes"
 
 {- Checks a symlink target to see if it appears to point to annexed content. -}
 isLinkToAnnex :: FilePath -> Bool
-isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s
+isLinkToAnnex s = ("/" ++ d) `isInfixOf` s || d `isPrefixOf` s
+	where
+		d = ".git" </> objectDir
 
 {- Converts a key into a filename fragment without any directory.
  -

Seek.hs

@@ -4,7 +4,7 @@
  - the values a user passes to a command, and prepare actions operating
  - on them.
  -
- - Copyright 2010-2011 Joey Hess <joey@kitenet.net>
+ - Copyright 2010-2012 Joey Hess <joey@kitenet.net>
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -41,6 +41,14 @@ withFilesNotInGit a params = do
 			g <- gitRepo
 			liftIO $ (\p -> LsFiles.notInRepo force p g) l
 
+withPathContents :: ((FilePath, FilePath) -> CommandStart) -> CommandSeek
+withPathContents a params = map a . concat <$> liftIO (mapM get params)
+	where
+		get p = ifM (isDirectory <$> getFileStatus p)
+			( map (\f -> (f, makeRelative p f)) <$> dirContentsRecursive p
+			, return [(p, takeFileName p)]
+			)
+
 withWords :: ([String] -> CommandStart) -> CommandSeek
 withWords a params = return [a params]
 

Utility/Directory.hs

@@ -17,6 +17,7 @@ import System.FilePath
 import Control.Applicative
 import Control.Exception (bracket_)
 import System.Posix.Directory
+import System.IO.Unsafe (unsafeInterleaveIO)
 
 import Utility.SafeCommand
 import Utility.TempFile
@@ -24,14 +25,37 @@ import Utility.Exception
 import Utility.Monad
 import Utility.Path
 
+dirCruft :: FilePath -> Bool
+dirCruft "." = True
+dirCruft ".." = True
+dirCruft _ = False
+
 {- Lists the contents of a directory.
  - Unlike getDirectoryContents, paths are not relative to the directory. -}
 dirContents :: FilePath -> IO [FilePath]
-dirContents d = map (d </>) . filter notcruft <$> getDirectoryContents d
+dirContents d = map (d </>) . filter (not . dirCruft) <$> getDirectoryContents d
+
+{- Gets contents of directory, and then its subdirectories, recursively,
+ - and lazily. -}
+dirContentsRecursive :: FilePath -> IO [FilePath]
+dirContentsRecursive topdir = dirContentsRecursive' topdir [""]
+
+dirContentsRecursive' :: FilePath -> [FilePath] -> IO [FilePath]
+dirContentsRecursive' _ [] = return []
+dirContentsRecursive' topdir (dir:dirs) = unsafeInterleaveIO $ do
+	(files, dirs') <- collect [] [] =<< dirContents (topdir </> dir)
+	files' <- dirContentsRecursive' topdir (dirs' ++ dirs)
+	return (files ++ files')
 	where
-		notcruft "." = False
-		notcruft ".." = False
-		notcruft _ = True
+		collect files dirs' [] = return (reverse files, reverse dirs')
+		collect files dirs' (entry:entries)
+			| dirCruft entry = collect files dirs' entries
+			| otherwise = do
+				let dirEntry = dir </> entry
+				ifM (doesDirectoryExist $ topdir </> dirEntry)
+					( collect files (dirEntry:dirs') entries
+					, collect (dirEntry:files) dirs' entries
+					)			
 
 {- Moves one filename to another.
  - First tries a rename, but falls back to moving across devices if needed. -}

debian/changelog

@@ -1,6 +1,11 @@
 git-annex (3.20120523) UNRELEASED; urgency=low
 
   * sync: Show a nicer message if a user tries to sync to a special remote.
+  * lock: Reset unlocked file to index, rather than to branch head.
+  * import: New subcommand, pulls files from a directory outside the annex
+    and adds them.
+  * Fix display of warning message when encountering a file that uses an
+    unsupported backend.
 
  -- Joey Hess <joeyh@debian.org>  Sun, 27 May 2012 20:55:29 -0400
 

doc/bugs/unlock_then_lock_of_uncommitted_file_loses_it.mdwn

@@ -1,5 +1,7 @@
-Add a file, then unlock it, and then lock it. There is an error and the
-symlink gets deleted. 
+Add a file (do not commit), then unlock it, and then lock it.
+There is an error and the symlink gets deleted.
 
 The file will still be staged in the index, and the file content is still
 in the annex. --[[Joey]] 
+
+[[done]]

doc/design/assistant.mdwn

@@ -9,7 +9,7 @@ Feel free to chip in with comments! --[[Joey]]
 
 * Month 1 "like dropbox": [[!traillink inotify]] [[!traillink syncing]]
 * Month 2 "shiny webapp": [[!traillink webapp]] [[!traillink progressbars]]
-* Month 3 "easy setup": [[!traillink configurators]]
+* Month 3 "easy setup": [[!traillink configurators]] [[!traillink pairing]]
 * Month 4 "polishing": [[!traillink cloud]] [[!traillink leftovers]]
 * Months 5-6 "9k bonus round": [[!traillink Android]] [[!traillink partial_content]]
 

doc/design/assistant/android.mdwn

@@ -14,6 +14,14 @@ available in the App Store.
 * git (not all git commands are needed,
   but core plumbing and a few like `git-add` are.)
 
+### Android specific features
+
+The app should be aware of power status, and avoid expensive background
+jobs when low on battery or run flat out when plugged in.
+
+The app should be aware of network status, and avoid expensive data
+transfers when not on wifi. This may need to be configurable.
+
 ### FAT sucks
 
 The main media partition will use some awful FAT filesystem format from
@@ -26,13 +34,16 @@ handle all of git's filenames.) Possible approaches to this follow.
 
 Keep only a bare git repo on Android. The app would then need to include
 a file browser to access the files in there, and adding a file would move
-it into the repo. 
+it into the repo.
 
 Not ideal.
 
+Could be improved some by registering git-annex as a file handling app on
+Android, allowing you to "send to" git-annex.
+
 #### implement git smudge filters
 
-See [[smudge]].
+See [[todo/smudge]].
 
 Difficult. Would make git-annex generally better.
 

doc/design/assistant/cloud.mdwn

@@ -4,7 +4,8 @@ LANS.
 
 ## more cloud providers
 
-Git-annex already supports several cloud providers via [[special_remotes]].
+Git-annex already supports storing large files in 
+several cloud providers via [[special_remotes]].
 More should be added, such as:
 
 * Google drive (attractive because it's free, only 5 gb tho)
@@ -26,3 +27,17 @@ That's also ok.
 
 git-annex will need a way to tell the difference between these, either
 heuristically, or via configuration.
+
+Also needed for USB keys and Android gadgets.
+
+## storing git repos in the cloud
+
+Of course, one option is to just use github etc to store the git repo.
+
+Two things can store git repos in Anazon S3:
+* <http://gabrito.com/post/storing-git-repositories-in-amazon-s3-for-high-availability>
+* <http://wiki.cs.pdx.edu/oss2009/index/projects/gits3.html>
+
+Another option is to not store the git repo in the cloud, but push/pull
+peer-to-peer. When peers cannot directly talk to one-another, this could be
+bounced through something like XMPP.

doc/design/assistant/configurators.mdwn

@@ -10,12 +10,7 @@ through setting up common use cases.
 * Create a repository (run when the web app is started without a configured
   repository too).
 * Clone this repo to a USB drive.
-* Clone this repo to another host:
-  1. Prompt for the hostname (or do avahi local machine discovery). 
-  2. Enable the two hosts to ssh to one-another and run git-annex shell.
-     (A tricky problem, if ssh keys need to be added to do that.)
-  3. Push over a clone of the repository. (Using git-annex-shell?)
-  4. Start [[syncing]].
+* Clone this repo to another host. (Needs [[pairing]])
 * Set up Amazon S3.
 * Set up rsync remote.
 * Set up encryption.

doc/design/assistant/inotify.mdwn

@@ -8,7 +8,9 @@ useful, it needs to:
 - notice deleted files and stage the deletion
   (tricky; there's a race with add..)
 - notice renamed files, auto-fix the symlink, and stage the new file location
-- periodically auto-commit staged changes
+- periodically auto-commit staged changes (avoid autocommitting when
+  lots of changes are coming in)
+- tunable delays before adding new files, etc
 - honor .gitignore, not adding files it excludesa
 
 Also to do:

doc/design/assistant/pairing.mdwn

@@ -0,0 +1,13 @@
+For git-annex to be able to clone its repo to another host, it'd be good to
+have some way of pairing devices.
+
+It could work like this:
+
+1. Prompt for the hostname (or do avahi local machine discovery). 
+2. Enable the two hosts to ssh to one-another and run git-annex shell.
+   (A tricky problem, if ssh keys need to be added to do that.)
+3. Push over a clone of the repository. (Using git-annex-shell?)
+4. Start [[syncing]].
+
+Also look into the method used by
+<https://support.mozilla.org/en-US/kb/add-a-device-to-firefox-sync>

doc/design/assistant/webapp.mdwn

@@ -7,6 +7,9 @@ The webapp is a web server that displays a shiny interface.
   token. This guards against other users on the same system.
 * I would like to avoid passwords or other authentication methods,
   it's your local system.
+* Alternative for Linux at least would be to write a small program using
+  GTK+ Webkit, that runs the webapp, and can know what user ran it, avoiding
+  needing authentication.
 
 ## interface
 
@@ -14,6 +17,12 @@ The webapp is a web server that displays a shiny interface.
 * progress bars for each file
 * drag and drop to reorder
 * cancel and pause
+* keep it usable w/o javascript, and accessible to blind, etc
+
+## other features
+
+* there could be a UI to export a file, which would make it be served up
+  over http by the web app
 
 ## implementation
 

doc/design/assistant/windows.mdwn

@@ -6,12 +6,19 @@ Apparently new versions of Windows have something very like symlinks.
 (Or really, 3 or so things not entirely unlike symlinks and all different.)
 Stackoverflow has some details.
 
+NTFS supports symbolic links two different ways: an [[!wikipedia NTFS symbolic link]] and an [[!wikipedia NTFS_junction_point]].  The former seems like the closest analogue to POSIX symlinks.
+
 Make git use them, as it (apparently) does not yet.
 
-(What **does** git do on Windows when it clones a repo with symlinks?)
+Currently, on Windows, git checks out symlinks as files containing the symlink
+target as their contents.
 
 ## POSIX
 
 Lots of ifdefs and pain to deal with POSIX calls in the code base.
 
-Or I could try to use Cywin.
+Or I could try to use Cygwin.
+
+## Deeper system integration
+
+[NTFS Reparse Points](http://msdn.microsoft.com/en-us/library/aa365503%28v=VS.85%29.aspx) allow a program to define how the OS will interpret a file or directory in arbitrary ways.  This requires writing a file system filter.

doc/forum/Getting_started_with_Amazon_S3.mdwn

@@ -0,0 +1,28 @@
+I'm just getting started with git-annex and trying to wrap my head around using it with Amazon S3. I am familiar with using git, but things are a bit different as we can't init a repo at S3 directly.
+
+I've followed http://git-annex.branchable.com/tips/using_Amazon_S3/, and performed:
+
+`git init`<br/>
+Initialized empty Git repository in /home/<br/>
+`git annex init`<br/>
+init  ok<br/>
+`git annex initremote s3 type=S3 encryption=FOOBAR bucket=foo`<br/>
+initremote s3 (encryption setup with gpg key YGTVT51715TFR) (checking bucket...) (gpg) ok<br/>
+`git annex describe s3 "Amazon S3"`<br/>
+describe s3 ok<br/>
+`git annexx add foo/`<br/>
+add foo/bar.txt<br/>
+add foo/bar.png<br/>
+...etc<br/>
+`git annex sync`<br/>
+51 files changed, 51 insertions(+)<br/>
+create mode 120000 foo/bar.txt<br/>
+create mode 120000 foo/bar.png<br/>
+...etc<br/>
+
+
+Looking at http://git-annex.branchable.com/git-annex/, I thought the files added would then be pushed to S3 by git annex sync, but that doesn't seem to be the case. I've also tried variations of got annex copy, like `git annex copy . --to s3`, without any luck.
+
+Is there a way to push to s3?
+
+Any help is appreciated!

doc/forum/Getting_started_with_Amazon_S3/comment_1_f50883133d5d4903cc95c0dcaa52d052._comment

@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.81.112"
+ subject="comment 1"
+ date="2012-05-29T19:09:50Z"
+ content="""
+`git annex sync` only syncs git metadata, not file contents, and metadata is not stored on S3, so it does notthing (much).
+
+`git annex move . --to s3` or `git annex copy . --to s3` is the right way to send the files to S3. I'm not sure why you say it's not working. I'd try it but Amazon is not letting me sign up for S3 again right now. Can you show what goes wrong with copy?
+"""]]

doc/forum/Getting_started_with_Amazon_S3/comment_2_e90aa3259d9a12cd67daa27d42d69ab5._comment

@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnoUOqs_lbuWyZBqyU6unHgUduJwDDgiKY"
+ nickname="Matt"
+ subject="comment 2"
+ date="2012-05-30T00:40:45Z"
+ content="""
+It's strange. I've done some testing on another machine, and this one, and the issue seems to be with adding only certain sub-directories of the git-annex directory. Would it cause an issue with git-annex if a sub-directory was a git repo?
+"""]]

doc/forum/Getting_started_with_Amazon_S3/comment_3_c3adce7c0f29e71ed9dd07103ede2c1a._comment

@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.81.112"
+ subject="comment 3"
+ date="2012-05-30T00:54:38Z"
+ content="""
+If the subdirectory has a .git, then it's a separate git repo, and inside the directory, all git (and git-annex) commands in it will operate on that nested repo and ignore the outside one.
+"""]]

doc/git-annex.mdwn

@@ -160,6 +160,15 @@ subdirectories).
   alternate locations from which the file can be downloaded. In this mode,
   addurl can be used both to add new files, or to add urls to existing files.
 
+* import [path ...]
+
+  Moves files from somewhere outside the git working copy, and adds them to
+  the annex. Individual files to import can be specified. 
+  If a directory is specified, all files in it are imported, and any
+  subdirectory structure inside it is preserved.
+
+	git annex import /media/camera/DCIM/
+
 # REPOSITORY SETUP COMMANDS
 
 * init [description]

doc/special_remotes/S3.mdwn

@@ -6,8 +6,8 @@ See [[tips/using_Amazon_S3]] and
 
 ## configuration
 
-The standard environment variables `ANNEX_S3_ACCESS_KEY_ID` and 
-`ANNEX_S3_SECRET_ACCESS_KEY` are used to supply login credentials
+The standard environment variables `AWS_ACCESS_KEY_ID` and 
+`AWS_SECRET_ACCESS_KEY` are used to supply login credentials
 for Amazon. When encryption is enabled, they are stored in encrypted form
 by `git annex initremote`, so you do not need to keep the environment
 variables set after the initial initalization of the remote.

doc/special_remotes/S3/comment_1_4a1f7a230dad6caa84831685b236fd73._comment

@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnoUOqs_lbuWyZBqyU6unHgUduJwDDgiKY"
+ nickname="Matt"
+ subject="environment variables"
+ date="2012-05-29T12:40:25Z"
+ content="""
+Just noting that the environment variables `ANNEX_S3_ACCESS_KEY_ID` and `ANNEX_S3_SECRET_ACCESS_KEY` seem to have been changed to `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
+"""]]

doc/special_remotes/S3/comment_2_5b22d67de946f4d34a4a3c7449d32988._comment

@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.81.112"
+ subject="comment 2"
+ date="2012-05-29T19:10:46Z"
+ content="""
+Thanks, I've fixed that. (You could have too.. this is a wiki ;)
+"""]]

doc/special_remotes/S3/comment_3_bcab2bd0f168954243aa9bcc9671bd94._comment

@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnoUOqs_lbuWyZBqyU6unHgUduJwDDgiKY"
+ nickname="Matt"
+ subject="comment 3"
+ date="2012-05-30T00:26:33Z"
+ content="""
+Thanks! Being new here, I didn't want to overstep my boundaries. I've gone ahead and made a small edit and will do so elsewhere as needed.
+"""]]

doc/tips/using_Amazon_S3.mdwn

@@ -4,8 +4,8 @@ Amazon S3, and use git-annex to transfer files into the cloud.
 
 First, export your S3 credentials:
 
-	# export ANNEX_S3_ACCESS_KEY_ID="08TJMT99S3511WOZEP91"
-	# export ANNEX_S3_SECRET_ACCESS_KEY="s3kr1t"
+	# export AWS_ACCESS_KEY_ID="08TJMT99S3511WOZEP91"
+	# export AWS_SECRET_ACCESS_KEY="s3kr1t"
 
 Now, create a gpg key, if you don't already have one. This will be used
 to encrypt everything stored in S3, for your privacy. Once you have

doc/tips/using_Amazon_S3/comment_1_666a26f95024760c99c627eed37b1966._comment

@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnoUOqs_lbuWyZBqyU6unHgUduJwDDgiKY"
+ nickname="Matt"
+ subject="ANNEX_S3 vs AWS for keys"
+ date="2012-05-29T12:24:25Z"
+ content="""
+The instructions state ANNEX_S3_ACCESS_KEY_ID and ANNEX_SECRET_ACCESS_KEY but git-annex cannot connect with those constants. git-annex tells me to set both \"AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY\" instead, which works. This is with Xubuntu 12.04.
+"""]]

doc/tips/using_Amazon_S3/comment_2_f5a0883be7dbb421b584c6dc0165f1ef._comment

@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.81.112"
+ subject="comment 2"
+ date="2012-05-29T19:10:42Z"
+ content="""
+Thanks, I've fixed that. (You could have too.. this is a wiki ;)
+"""]]

doc/tips/what_to_do_when_you_lose_a_repository/comment_1_cf19b8dc304dc37c26717174c4a98aa4._comment

@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://dlaxalde.myopenid.com/"
+ nickname="dl"
+ subject="comment 1"
+ date="2012-05-31T14:36:33Z"
+ content="""
+Is there a way to have git-annex completely ignore a repository? I see that
+the `dead` command adds the uuid of the repository to `trust.log` but does
+not change `uuid.log`. Is it enough to remove the corresponding line in
+`uuid.log` and `trust.log`?
+"""]]

doc/tips/what_to_do_when_you_lose_a_repository/comment_3_fa9ca81668f5faebf2f61b10f82c97d2._comment

@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.243"
+ subject="comment 3"
+ date="2012-05-31T17:01:37Z"
+ content="""
+`dead` is the best we can do. The automatic merging used on the git-annex branch tends to re-add lines that are deleted in one repo when merging with another that still has them.
+"""]]