Merge branch 'master' of ssh://git-annex.branchable.com

Conflicts:
	doc/tips/using_Amazon_S3.mdwn

Backend.hs

@@ -75,16 +75,16 @@ genKey' (b:bs) file = do
  - by examining what the file symlinks to. -}
 lookupFile :: FilePath -> Annex (Maybe (Key, Backend))
 lookupFile file = do
-	tl <- liftIO $ tryIO getsymlink
+	tl <- liftIO $ tryIO $ readSymbolicLink file
 	case tl of
 		Left _ -> return Nothing
 		Right l -> makekey l
 	where
-		getsymlink = takeFileName <$> readSymbolicLink file
+		makekey l = maybe (return Nothing) (makeret l) (fileKey $ takeFileName l)
-		makekey l = maybe (return Nothing) (makeret l) (fileKey l)
 		makeret l k = let bname = keyBackendName k in
 			case maybeLookupBackendName bname of
-				Just backend -> return $ Just (k, backend)
+				Just backend -> do
+					return $ Just (k, backend)
 				Nothing -> do
 					when (isLinkToAnnex l) $ warning $
 						"skipping " ++ file ++

Backend/SHA.hs

@@ -45,7 +45,7 @@ genBackendE size =
 
 shaCommand :: SHASize -> Maybe String
 shaCommand 1 = SysConfig.sha1
-shaCommand 256 = SysConfig.sha256
+shaCommand 256 = Just SysConfig.sha256
 shaCommand 224 = SysConfig.sha224
 shaCommand 384 = SysConfig.sha384
 shaCommand 512 = SysConfig.sha512

Build/Configure.hs

@@ -6,6 +6,7 @@ import System.Directory
 import Data.List
 import System.Cmd.Utils
 import Control.Applicative
+import System.FilePath
 
 import Build.TestConfig
 import Utility.SafeCommand
@@ -26,15 +27,21 @@ tests =
 	, TestCase "bup" $ testCmd "bup" "bup --version >/dev/null"
 	, TestCase "gpg" $ testCmd "gpg" "gpg --version >/dev/null"
 	, TestCase "ssh connection caching" getSshConnectionCaching
-	] ++ shaTestCases [1, 256, 512, 224, 384]
+	] ++ shaTestCases False [1, 512, 224, 384] ++ shaTestCases True [256]
 
-shaTestCases :: [Int] -> [TestCase]
+shaTestCases :: Bool -> [Int] -> [TestCase]
-shaTestCases l = map make l
+shaTestCases required l = map make l
-	where make n =
+	where
-		let
+		make n = TestCase key $ selector key (shacmds n) "</dev/null"
-			cmds = map (\x -> "sha" ++ show n ++ x) ["", "sum"]
+			where
-			key = "sha" ++ show n
+				key = "sha" ++ show n
-		in TestCase key $ maybeSelectCmd key cmds "</dev/null"
+		selector = if required then selectCmd else maybeSelectCmd
+		shacmds n = concatMap (\x -> [x, osxpath </> x]) $
+			map (\x -> "sha" ++ show n ++ x) ["", "sum"]
+		-- Max OSX puts GNU tools outside PATH, so look in
+		-- the location it uses, and remember where to run them
+		-- from.
+		osxpath = "/opt/local/libexec/gnubin"
 
 tmpDir :: String
 tmpDir = "tmp"

Command/Import.hs

@@ -0,0 +1,39 @@
+{- git-annex command
+ -
+ - Copyright 2012 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.Import where
+
+import Common.Annex
+import Command
+import qualified Annex
+import qualified Command.Add
+
+def :: [Command]
+def = [command "import" paramPaths seek "move and add files from outside git working copy"]
+
+seek :: [CommandSeek]
+seek = [withPathContents start]
+
+start :: (FilePath, FilePath) -> CommandStart
+start (srcfile, destfile) = notBareRepo $
+	ifM (liftIO $ isRegularFile <$> getSymbolicLinkStatus srcfile)
+		( do
+			showStart "import" destfile
+			next $ perform srcfile destfile
+		, stop
+		)
+
+perform :: FilePath -> FilePath -> CommandPerform
+perform srcfile destfile = do
+	whenM (liftIO $ doesFileExist destfile) $
+		unlessM (Annex.getState Annex.force) $
+			error $ "not overwriting existing " ++ destfile ++
+				" (use --force to override)"
+ 
+	liftIO $ createDirectoryIfMissing True (parentDir destfile)
+	liftIO $ moveFile srcfile destfile
+	Command.Add.perform destfile

Command/Lock.hs

@@ -24,9 +24,5 @@ start file = do
 
 perform :: FilePath -> CommandPerform
 perform file = do
-	liftIO $ removeFile file
+	Annex.Queue.add "checkout" [Param "--"] [file]
-	-- Checkout from HEAD to get rid of any changes that might be 
-	-- staged in the index, and get back to the previous symlink to
-	-- the content.
-	Annex.Queue.add "checkout" [Param "HEAD", Param "--"] [file]
 	next $ return True -- no cleanup needed

GitAnnex.hs

@@ -54,6 +54,7 @@ import qualified Command.Semitrust
 import qualified Command.Dead
 import qualified Command.Sync
 import qualified Command.AddUrl
+import qualified Command.Import
 import qualified Command.Map
 import qualified Command.Upgrade
 import qualified Command.Version
@@ -69,6 +70,7 @@ cmds = concat
 	, Command.Lock.def
 	, Command.Sync.def
 	, Command.AddUrl.def
+	, Command.Import.def
 	, Command.Init.def
 	, Command.Describe.def
 	, Command.InitRemote.def

Locations.hs

@@ -155,7 +155,9 @@ gitAnnexRemotesDir r = addTrailingPathSeparator $ gitAnnexDir r </> "remotes"
 
 {- Checks a symlink target to see if it appears to point to annexed content. -}
 isLinkToAnnex :: FilePath -> Bool
-isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s
+isLinkToAnnex s = ("/" ++ d) `isInfixOf` s || d `isPrefixOf` s
+	where
+		d = ".git" </> objectDir
 
 {- Converts a key into a filename fragment without any directory.
  -

Seek.hs

@@ -4,7 +4,7 @@
  - the values a user passes to a command, and prepare actions operating
  - on them.
  -
- - Copyright 2010-2011 Joey Hess <joey@kitenet.net>
+ - Copyright 2010-2012 Joey Hess <joey@kitenet.net>
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -41,6 +41,14 @@ withFilesNotInGit a params = do
 			g <- gitRepo
 			liftIO $ (\p -> LsFiles.notInRepo force p g) l
 
+withPathContents :: ((FilePath, FilePath) -> CommandStart) -> CommandSeek
+withPathContents a params = map a . concat <$> liftIO (mapM get params)
+	where
+		get p = ifM (isDirectory <$> getFileStatus p)
+			( map (\f -> (f, makeRelative p f)) <$> dirContentsRecursive p
+			, return [(p, takeFileName p)]
+			)
+
 withWords :: ([String] -> CommandStart) -> CommandSeek
 withWords a params = return [a params]
 

Utility/Directory.hs

@@ -17,6 +17,7 @@ import System.FilePath
 import Control.Applicative
 import Control.Exception (bracket_)
 import System.Posix.Directory
+import System.IO.Unsafe (unsafeInterleaveIO)
 
 import Utility.SafeCommand
 import Utility.TempFile
@@ -24,14 +25,37 @@ import Utility.Exception
 import Utility.Monad
 import Utility.Path
 
+dirCruft :: FilePath -> Bool
+dirCruft "." = True
+dirCruft ".." = True
+dirCruft _ = False
+
 {- Lists the contents of a directory.
  - Unlike getDirectoryContents, paths are not relative to the directory. -}
 dirContents :: FilePath -> IO [FilePath]
-dirContents d = map (d </>) . filter notcruft <$> getDirectoryContents d
+dirContents d = map (d </>) . filter (not . dirCruft) <$> getDirectoryContents d
+
+{- Gets contents of directory, and then its subdirectories, recursively,
+ - and lazily. -}
+dirContentsRecursive :: FilePath -> IO [FilePath]
+dirContentsRecursive topdir = dirContentsRecursive' topdir [""]
+
+dirContentsRecursive' :: FilePath -> [FilePath] -> IO [FilePath]
+dirContentsRecursive' _ [] = return []
+dirContentsRecursive' topdir (dir:dirs) = unsafeInterleaveIO $ do
+	(files, dirs') <- collect [] [] =<< dirContents (topdir </> dir)
+	files' <- dirContentsRecursive' topdir (dirs' ++ dirs)
+	return (files ++ files')
 	where
-		notcruft "." = False
+		collect files dirs' [] = return (reverse files, reverse dirs')
-		notcruft ".." = False
+		collect files dirs' (entry:entries)
-		notcruft _ = True
+			| dirCruft entry = collect files dirs' entries
+			| otherwise = do
+				let dirEntry = dir </> entry
+				ifM (doesDirectoryExist $ topdir </> dirEntry)
+					( collect files (dirEntry:dirs') entries
+					, collect (dirEntry:files) dirs' entries
+					)			
 
 {- Moves one filename to another.
  - First tries a rename, but falls back to moving across devices if needed. -}

debian/changelog

@@ -1,6 +1,13 @@
 git-annex (3.20120523) UNRELEASED; urgency=low
 
   * sync: Show a nicer message if a user tries to sync to a special remote.
+  * lock: Reset unlocked file to index, rather than to branch head.
+  * import: New subcommand, pulls files from a directory outside the annex
+    and adds them.
+  * Fix display of warning message when encountering a file that uses an
+    unsupported backend.
+  * Require that the SHA256 backend can be used when building, since it's the
+    default.
 
  -- Joey Hess <joeyh@debian.org>  Sun, 27 May 2012 20:55:29 -0400
 

doc/bugs/unlock_then_lock_of_uncommitted_file_loses_it.mdwn

@@ -1,5 +1,7 @@
-Add a file, then unlock it, and then lock it. There is an error and the
+Add a file (do not commit), then unlock it, and then lock it.
-symlink gets deleted. 
+There is an error and the symlink gets deleted.
 
 The file will still be staged in the index, and the file content is still
 in the annex. --[[Joey]] 
+
+[[done]]

doc/design/assistant.mdwn

@@ -9,7 +9,7 @@ Feel free to chip in with comments! --[[Joey]]
 
 * Month 1 "like dropbox": [[!traillink inotify]] [[!traillink syncing]]
 * Month 2 "shiny webapp": [[!traillink webapp]] [[!traillink progressbars]]
-* Month 3 "easy setup": [[!traillink configurators]]
+* Month 3 "easy setup": [[!traillink configurators]] [[!traillink pairing]]
 * Month 4 "polishing": [[!traillink cloud]] [[!traillink leftovers]]
 * Months 5-6 "9k bonus round": [[!traillink Android]] [[!traillink partial_content]]
 

doc/design/assistant/android.mdwn

@@ -14,6 +14,14 @@ available in the App Store.
 * git (not all git commands are needed,
   but core plumbing and a few like `git-add` are.)
 
+### Android specific features
+
+The app should be aware of power status, and avoid expensive background
+jobs when low on battery or run flat out when plugged in.
+
+The app should be aware of network status, and avoid expensive data
+transfers when not on wifi. This may need to be configurable.
+
 ### FAT sucks
 
 The main media partition will use some awful FAT filesystem format from
@@ -26,13 +34,16 @@ handle all of git's filenames.) Possible approaches to this follow.
 
 Keep only a bare git repo on Android. The app would then need to include
 a file browser to access the files in there, and adding a file would move
-it into the repo. 
+it into the repo.
 
 Not ideal.
 
+Could be improved some by registering git-annex as a file handling app on
+Android, allowing you to "send to" git-annex.
+
 #### implement git smudge filters
 
-See [[smudge]].
+See [[todo/smudge]].
 
 Difficult. Would make git-annex generally better.
 

doc/design/assistant/comment_1_a48fcfbf97f0a373ea375cd8f07f0fc8._comment

@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 1"
+ date="2012-06-02T12:06:37Z"
+ content="""
+Will statically linked binaries be provided for say Linux, OSX and *BSD?  I think having some statically linked binaries will certainly help and appeal to a lot of users.
+"""]]

doc/design/assistant/configurators.mdwn

@@ -10,12 +10,7 @@ through setting up common use cases.
 * Create a repository (run when the web app is started without a configured
   repository too).
 * Clone this repo to a USB drive.
-* Clone this repo to another host:
+* Clone this repo to another host. (Needs [[pairing]])
-  1. Prompt for the hostname (or do avahi local machine discovery). 
-  2. Enable the two hosts to ssh to one-another and run git-annex shell.
-     (A tricky problem, if ssh keys need to be added to do that.)
-  3. Push over a clone of the repository. (Using git-annex-shell?)
-  4. Start [[syncing]].
 * Set up Amazon S3.
 * Set up rsync remote.
 * Set up encryption.

doc/design/assistant/inotify.mdwn

@@ -8,7 +8,9 @@ useful, it needs to:
 - notice deleted files and stage the deletion
   (tricky; there's a race with add..)
 - notice renamed files, auto-fix the symlink, and stage the new file location
-- periodically auto-commit staged changes
+- periodically auto-commit staged changes (avoid autocommitting when
+  lots of changes are coming in)
+- tunable delays before adding new files, etc
 - honor .gitignore, not adding files it excludesa
 
 Also to do:

doc/design/assistant/pairing.mdwn

@@ -0,0 +1,13 @@
+For git-annex to be able to clone its repo to another host, it'd be good to
+have some way of pairing devices.
+
+It could work like this:
+
+1. Prompt for the hostname (or do avahi local machine discovery). 
+2. Enable the two hosts to ssh to one-another and run git-annex shell.
+   (A tricky problem, if ssh keys need to be added to do that.)
+3. Push over a clone of the repository. (Using git-annex-shell?)
+4. Start [[syncing]].
+
+Also look into the method used by
+<https://support.mozilla.org/en-US/kb/add-a-device-to-firefox-sync>

doc/design/assistant/webapp.mdwn

@@ -7,6 +7,9 @@ The webapp is a web server that displays a shiny interface.
   token. This guards against other users on the same system.
 * I would like to avoid passwords or other authentication methods,
   it's your local system.
+* Alternative for Linux at least would be to write a small program using
+  GTK+ Webkit, that runs the webapp, and can know what user ran it, avoiding
+  needing authentication.
 
 ## interface
 
@@ -16,6 +19,11 @@ The webapp is a web server that displays a shiny interface.
 * cancel and pause
 * keep it usable w/o javascript, and accessible to blind, etc
 
+## other features
+
+* there could be a UI to export a file, which would make it be served up
+  over http by the web app
+
 ## implementation
 
 Hope to use Yesod.

doc/design/assistant/windows.mdwn

@@ -6,12 +6,19 @@ Apparently new versions of Windows have something very like symlinks.
 (Or really, 3 or so things not entirely unlike symlinks and all different.)
 Stackoverflow has some details.
 
+NTFS supports symbolic links two different ways: an [[!wikipedia NTFS symbolic link]] and an [[!wikipedia NTFS_junction_point]].  The former seems like the closest analogue to POSIX symlinks.
+
 Make git use them, as it (apparently) does not yet.
 
-(What **does** git do on Windows when it clones a repo with symlinks?)
+Currently, on Windows, git checks out symlinks as files containing the symlink
+target as their contents.
 
 ## POSIX
 
 Lots of ifdefs and pain to deal with POSIX calls in the code base.
 
-Or I could try to use Cywin.
+Or I could try to use Cygwin.
+
+## Deeper system integration
+
+[NTFS Reparse Points](http://msdn.microsoft.com/en-us/library/aa365503%28v=VS.85%29.aspx) allow a program to define how the OS will interpret a file or directory in arbitrary ways.  This requires writing a file system filter.

doc/forum/Getting_started_with_Amazon_S3/comment_2_e90aa3259d9a12cd67daa27d42d69ab5._comment

@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnoUOqs_lbuWyZBqyU6unHgUduJwDDgiKY"
+ nickname="Matt"
+ subject="comment 2"
+ date="2012-05-30T00:40:45Z"
+ content="""
+It's strange. I've done some testing on another machine, and this one, and the issue seems to be with adding only certain sub-directories of the git-annex directory. Would it cause an issue with git-annex if a sub-directory was a git repo?
+"""]]

doc/forum/Getting_started_with_Amazon_S3/comment_3_c3adce7c0f29e71ed9dd07103ede2c1a._comment

@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.81.112"
+ subject="comment 3"
+ date="2012-05-30T00:54:38Z"
+ content="""
+If the subdirectory has a .git, then it's a separate git repo, and inside the directory, all git (and git-annex) commands in it will operate on that nested repo and ignore the outside one.
+"""]]

doc/git-annex.mdwn

@@ -160,6 +160,15 @@ subdirectories).
   alternate locations from which the file can be downloaded. In this mode,
   addurl can be used both to add new files, or to add urls to existing files.
 
+* import [path ...]
+
+  Moves files from somewhere outside the git working copy, and adds them to
+  the annex. Individual files to import can be specified. 
+  If a directory is specified, all files in it are imported, and any
+  subdirectory structure inside it is preserved.
+
+	git annex import /media/camera/DCIM/
+
 # REPOSITORY SETUP COMMANDS
 
 * init [description]

doc/install/OSX.mdwn

@@ -7,9 +7,6 @@ sudo port install git-core ossp-uuid md5sha1sum coreutils pcre
 
 sudo ln -s /opt/local/include/pcre.h  /usr/include/pcre.h # This is hack that allows pcre-light to find pcre
 
-# optional: this will enable the gnu tools, (to give sha224sum etc..., it does not override the BSD userland)
-export PATH=$PATH:/opt/local/libexec/gnubin
-
 git clone git://git-annex.branchable.com/ git-annex
 cd git-annex
 git checkout ghc7.0

doc/install/OSX/comment_5_50777853f808d57b957f8ce9a0f84b3d._comment

@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnHrjHxJAm39x8DR4bnbazQO6H0nMNuY9c"
+ nickname="Damien"
+ subject="sha256"
+ date="2012-06-01T16:13:05Z"
+ content="""
+If you're missing the `sha256sum` command with Homebrew, it's provided by `coreutils`. You have to change your `$PATH` before running `cabal install git-annex.cabal`:
+
+    PATH=\"$(brew --prefix coreutils)/libexec/gnubin:$PATH\"
+"""]]

doc/install/OSX/comment_6_18a8df794aa0ddd294dbf17d3d4c7fe2._comment

@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 6"
+ date="2012-06-01T17:24:29Z"
+ content="""
+Last night I made it look in /opt/local/libexec/gnubin .. if there's another directory it could look in, let me know. I am reluctant to make it run the brew command directly.
+"""]]

doc/special_remotes/S3.mdwn

@@ -6,8 +6,8 @@ See [[tips/using_Amazon_S3]] and
 
 ## configuration
 
-The standard environment variables `AWS_S3_ACCESS_KEY_ID` and 
+The standard environment variables `AWS_ACCESS_KEY_ID` and 
-`AWS_S3_SECRET_ACCESS_KEY` are used to supply login credentials
+`AWS_SECRET_ACCESS_KEY` are used to supply login credentials
 for Amazon. When encryption is enabled, they are stored in encrypted form
 by `git annex initremote`, so you do not need to keep the environment
 variables set after the initial initalization of the remote.

doc/special_remotes/S3/comment_3_bcab2bd0f168954243aa9bcc9671bd94._comment

@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnoUOqs_lbuWyZBqyU6unHgUduJwDDgiKY"
+ nickname="Matt"
+ subject="comment 3"
+ date="2012-05-30T00:26:33Z"
+ content="""
+Thanks! Being new here, I didn't want to overstep my boundaries. I've gone ahead and made a small edit and will do so elsewhere as needed.
+"""]]

doc/testimonials.mdwn

@@ -9,6 +9,13 @@
 <script src="//platform.twitter.com/widgets.js" charset="utf-8"></script>
 </div>
 
+<blockquote>
+What excites me about GIT ANNEX is how it fundamentally tracks the
+backup and availability of any data you own, and allows you to share
+data with a large or small audience, ensuring that the data survives.
+</blockquote>
+-- Jason Scott <http://ascii.textfiles.com/archives/3625>
+
 Seen on IRC:
 <pre>
 oh my god, git-annex is amazing

doc/tips/using_Amazon_S3.mdwn

@@ -4,7 +4,7 @@ Amazon S3, and use git-annex to transfer files into the cloud.
 
 First, export your S3 credentials:
 
-	# export AWS_S3_ACCESS_KEY_ID="08TJMT99S3511WOZEP91"
+	# export AWS_ACCESS_KEY_ID="08TJMT99S3511WOZEP91"
 	# export AWS_SECRET_ACCESS_KEY="s3kr1t"
 
 Now, create a gpg key, if you don't already have one. This will be used

doc/tips/what_to_do_when_you_lose_a_repository/comment_1_cf19b8dc304dc37c26717174c4a98aa4._comment

@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://dlaxalde.myopenid.com/"
+ nickname="dl"
+ subject="comment 1"
+ date="2012-05-31T14:36:33Z"
+ content="""
+Is there a way to have git-annex completely ignore a repository? I see that
+the `dead` command adds the uuid of the repository to `trust.log` but does
+not change `uuid.log`. Is it enough to remove the corresponding line in
+`uuid.log` and `trust.log`?
+"""]]

doc/tips/what_to_do_when_you_lose_a_repository/comment_3_fa9ca81668f5faebf2f61b10f82c97d2._comment

@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.243"
+ subject="comment 3"
+ date="2012-05-31T17:01:37Z"
+ content="""
+`dead` is the best we can do. The automatic merging used on the git-annex branch tends to re-add lines that are deleted in one repo when merging with another that still has them.
+"""]]