Merge branch 'master' into windows

Annex.hs

@@ -103,7 +103,7 @@ data AnnexState = AnnexState
 	, auto :: Bool
 	, branchstate :: BranchState
 	, repoqueue :: Maybe Git.Queue.Queue
-	, catfilehandle :: Maybe CatFileHandle
+	, catfilehandles :: M.Map FilePath CatFileHandle
 	, checkattrhandle :: Maybe CheckAttrHandle
 	, forcebackend :: Maybe String
 	, limit :: Matcher (FileInfo -> Annex Bool)
@@ -133,7 +133,7 @@ newState gitrepo = AnnexState
 	, auto = False
 	, branchstate = startBranchState
 	, repoqueue = Nothing
-	, catfilehandle = Nothing
+	, catfilehandles = M.empty
 	, checkattrhandle = Nothing
 	, forcebackend = Nothing
 	, limit = Left []

Annex/CatFile.hs

@@ -1,6 +1,6 @@
 {- git cat-file interface, with handle automatically stored in the Annex monad
  -
- - Copyright 2011 Joey Hess <joey@kitenet.net>
+ - Copyright 2011-2013 Joey Hess <joey@kitenet.net>
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -15,6 +15,7 @@ module Annex.CatFile (
 ) where
 
 import qualified Data.ByteString.Lazy as L
+import qualified Data.Map as M
 
 import Common.Annex
 import qualified Git
@@ -38,13 +39,20 @@ catObjectDetails ref = do
 	h <- catFileHandle
 	liftIO $ Git.CatFile.catObjectDetails h ref
 
+{- There can be multiple index files, and a different cat-file is needed
+ - for each. This is selected by setting GIT_INDEX_FILE in the gitEnv. -}
 catFileHandle :: Annex Git.CatFile.CatFileHandle
-catFileHandle = maybe startup return =<< Annex.getState Annex.catfilehandle
+catFileHandle = do
-  where
+	m <- Annex.getState Annex.catfilehandles
-	startup = do
+	indexfile <- fromMaybe "" . maybe Nothing (lookup "GIT_INDEX_FILE")
-		h <- inRepo Git.CatFile.catFileStart
+		<$> fromRepo gitEnv
-		Annex.changeState $ \s -> s { Annex.catfilehandle = Just h }
+	case M.lookup indexfile m of
-		return h
+		Just h -> return h
+		Nothing -> do
+			h <- inRepo Git.CatFile.catFileStart
+			let m' = M.insert indexfile h m
+			Annex.changeState $ \s -> s { Annex.catfilehandles = m' }
+			return h
 
 {- From the Sha or Ref of a symlink back to the key. -}
 catKey :: Ref -> Annex (Maybe Key)
@@ -57,7 +65,7 @@ catKey ref = do
 {- From a file in git back to the key.
  -
  - Prefixing the file with ./ makes this work even if in a subdirectory
- - of a repo. For some reason, HEAD is sometimes needed.
+ - of a repo.
  -}
 catKeyFile :: FilePath -> Annex (Maybe Key)
-catKeyFile f = catKey $ Ref $ "HEAD:./" ++ f
+catKeyFile f = catKey $ Ref $ ":./" ++ f

Annex/Direct.hs

@@ -191,6 +191,7 @@ toDirectGen k f = do
 		{- Move content from annex to direct file. -}
 		thawContentDir loc
 		updateInodeCache k loc
+		addAssociatedFile k f
 		thawContent loc
 		replaceFile f $ liftIO . moveFile loc
 	fromdirect = do

debian/changelog

@@ -6,6 +6,8 @@ git-annex (4.20130502) UNRELEASED; urgency=low
   * git-annex-shell: Ensure that received files can be read. Files
     transferred from some Android devices may have very broken permissions
     as received.
+  * direct mode: Direct mode commands now work on files staged in the index,
+    they do not need to be committed to git.
   * Temporarily add an upper bound to the version of yesod that can be built
     with, since yesod 1.2 has a great many changes that will require extensive
     work on the webapp.
@@ -24,6 +26,7 @@ git-annex (4.20130502) UNRELEASED; urgency=low
     the destination's default file permissions, as the local ones can
     be arbitrarily broken. (Ie, ----rwxr-x for files on Android)
   * migrate: Detect if a file gets corrupted while it's being migrated.
+  * Debian: Add a menu file.
 
  -- Joey Hess <joeyh@debian.org>  Thu, 02 May 2013 20:39:19 -0400
 

debian/menu

@@ -0,0 +1,2 @@
+?package(git-annex):needs="X11" section="Applications/Network" \
+	title="git-annex assistant" command="git annex webapp"

doc/bugs/Adding_box.com_remote_on_Android_fails_for_me.mdwn

@@ -0,0 +1,17 @@
+### Please describe the problem.
+
+After submitting the form in the webapp for adding a box.com remote, I get:
+
+   Internal Server Error - WEBDAV failed to write file: "Unauthorized": user error
+
+### What steps will reproduce the problem?
+
+Fill in the box.com add remote form. Username=username, password=password, "share..."=checked, directory=annex, Encryption="Encrypt all data" and hit the "Add repository" button.
+
+### What version of git-annex are you using? On what operating system?
+
+  git-annex version 4.20130513-g5185533 on Android 4.2.2
+
+### Please provide any additional information below.
+
+Didn't find a .git/annex/debug.log

doc/bugs/Adding_box.com_remote_on_Android_fails_for_me/comment_1_0303ce880415d7e043533551c2b24694._comment

@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmmLfBDEDFfEOba8Ra46nsnTmoNHFsLObo"
+ nickname="Brian"
+ subject="It worked my second try"
+ date="2013-05-14T00:39:08Z"
+ content="""
+I enabled debug logging and tried again and this time the remote was successfully added! Maybe the first time I mistyped my password?
+
+Either way, it is working now and I created a file in my annex directory and the assistant successfully synced it to my box.com repo.
+"""]]

doc/bugs/Browser_fails_to_launch_on_Android___39__git_annex_webapp__39__.mdwn

@@ -0,0 +1,33 @@
+### Please describe the problem.
+
+When I launch git annex Android application, it fails to open the web browser. Workaround is to copy the url and manually paste it into web browser url line.
+
+### What steps will reproduce the problem?
+
+See above
+
+### What version of git-annex are you using? On what operating system?
+
+git-annex version 4.20130513-g5185533 on Android 4.2.2
+
+### Please provide any additional information below.
+
+[[!format sh """
+u0_a126@android:/sdcard/git-annex.home $ git annex webapp
+Launching web browser on http://127.0.0.1:47557/?auth=[...snip...]
+Starting: Intent { act=android.intent.action.VIEW dat=http://127.0.0.1:47557/?auth=[...snip...] }
+java.lang.SecurityException: Permission Denial: startActivity asks to run as user -2 but is calling from user 0; this requires android.permission.INTERACT_ACROSS_USERS_FULL
+        at android.os.Parcel.readException(Parcel.java:1425)
+        at android.os.Parcel.readException(Parcel.java:1379)
+        at android.app.ActivityManagerProxy.startActivityAsUser(ActivityManagerNative.java:1921)
+        at com.android.commands.am.Am.runStart(Am.java:494)
+        at com.android.commands.am.Am.run(Am.java:109)
+        at com.android.commands.am.Am.main(Am.java:82)
+        at com.android.internal.os.RuntimeInit.nativeFinishInit(Native Method)
+        at com.android.internal.os.RuntimeInit.main(RuntimeInit.java:235)
+        at dalvik.system.NativeStart.main(Native Method)
+failed to start web browser
+u0_a126@android:/sdcard/git-annex.home $
+
+# End of transcript or log.
+"""]]

doc/bugs/Browser_fails_to_launch_on_Android___39__git_annex_webapp__39__/comment_1_173393b0b3d2d8c622c0d8a2eaace421._comment

@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmmLfBDEDFfEOba8Ra46nsnTmoNHFsLObo"
+ nickname="Brian"
+ subject="Duplicate bug report"
+ date="2013-05-15T02:35:34Z"
+ content="""
+This report is a duplicate of [[Android app permission denial on startup]]
+"""]]

doc/design/assistant/blog/day_259__Android_dominos_toppling/comment_2_1ebc5aff5d217e1392cb7c8bb6c5156b._comment

@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmmLfBDEDFfEOba8Ra46nsnTmoNHFsLObo"
+ nickname="Brian"
+ subject="I also see the connection timed out"
+ date="2013-05-15T00:19:34Z"
+ content="""
+Same Android device and same behavior as the above comment. While the connection attempt was being made, I switched to the terminal window and saw a connection to 173.194.46.86:5222 in the SYN_SENT state (and that connection wasn't there prior to the jabber configuration attempt).
+
+Even from my desktop, manual connection attempts to 173.194.46.86:5222 timed out.
+
+Then I ran the same jabber configuration from my desktop and it worked fin. Using netstat, I could see it connected to 74.125.133.125:5222.
+
+So is the Android version using the wrong address?
+"""]]

doc/design/assistant/blog/day_263_catching_up.mdwn

@@ -7,3 +7,5 @@ working. But nothing that moves content quite works yet..
 I've run into a stumbling block with `rsync`. It thinks that
 `C:\repo` is a path on a ssh server named "C". Seems I will need to translate
 native windows paths to unix-style paths when running rsync.
+
+[[!meta date="13 May 2013"]]

doc/design/assistant/blog/day_263_catching_up/comment_1_9023da0573dfc81644d68128adb331a7._comment

@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkHscTHMCNvjJ6nLI1VpsBrJFI5FTwhUT4"
+ nickname="David"
+ subject="Just use Unix Paths like git does?"
+ date="2013-05-15T20:43:34Z"
+ content="""
+Can't you just use unix-style paths for your internal data structures, and wrap/patch the filesystem I/O code to convert between the two?
+"""]]

doc/design/assistant/blog/day_264__Windows_second_stage_complete.mdwn

@@ -0,0 +1,21 @@
+The Windows port can now do everything in the [[walkthrough]]. It can use
+both local and remote git repositories. Some special remotes work
+(directory at least; probably rsync; likely any other special remote that
+can have its dependencies built). Missing features include most special
+remotes, gpg encryption, and of course, the assistant.
+
+Also built a NullSoft installer for git-annex today. This was made very
+easy when I found the Haskell ncis library, which provides a DSL embedding
+the language used to write NullSoft installers into Haskell. So I didn't
+need to learn a new language, yay! And could pull in all my helpful
+Haskell utility libraries in the program that builds the installer.
+
+The only tricky part was: How to get git-annex onto PATH? The standard way
+to do this seems to be to use a multiple-hundred line include file. Of
+course, that file does not have any declared license..  Instead of that,
+I used a hack. The git installer for Windows adds itself to PATH, and is
+a pre-requisite for git-annex. So the git-annex installer just installs
+it into the same directory as git.
+
+So.. I'll be including this first stage Windows port, with installer in
+the next release. Anyone want to run a Windows autobuilder?

doc/design/assistant/blog/day_264__Windows_second_stage_complete/comment_1_42a7502d6ece75520eb59a76fdb1e2f0._comment

@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlJ2utMQgMEYAOs3Dfc6eZRyUzt4acNXUU"
+ nickname="David"
+ subject="git on windows"
+ date="2013-05-15T04:26:48Z"
+ content="""
+Adding git to the PATH variable is an optional step in the msysgit installer, fyi.
+
+"""]]

doc/design/assistant/blog/day_264__Windows_second_stage_complete/comment_2_f2b11322ac87e2a36cddc035b2c3c1ea._comment

@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 2"
+ date="2013-05-15T22:30:05Z"
+ content="""
+I was thinking maybe migrating away from gitbuilder to using something like buildbot as it seems to be a better fit for this project than gitbuilder.
+"""]]

doc/design/assistant/blog/day_264__Windows_second_stage_complete/comment_3_ea6ee05acb946fc7e8d95e62647cfa2a._comment

@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ nickname="joey"
+ subject="comment 3"
+ date="2013-05-15T22:56:27Z"
+ content="""
+@Jimmy, would be fine with me if it works better
+"""]]

doc/design/assistant/polls/Android_default_directory.mdwn

@@ -4,4 +4,4 @@ Same as the desktop webapp, users will be able to enter a directory they
 want the first time they run it, but to save typing on android, anything
 that gets enough votes will be included in a list of choices as well.
 
-[[!poll open=yes expandable=yes 43 "/sdcard/annex" 3 "Whole /sdcard" 4 "DCIM directory (photos and videos only)" 1 "Same as for regular git-annex. ~/annex/"]]
+[[!poll open=yes expandable=yes 44 "/sdcard/annex" 3 "Whole /sdcard" 4 "DCIM directory (photos and videos only)" 1 "Same as for regular git-annex. ~/annex/"]]

doc/forum/Accessing_files_in_bare_repository/comment_12_0e7ea5161b6da6e9bb9425bdb953de33._comment

@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/bBy7WkgQicYHIiiyj.Vm0TcMbxi2quzbPFef#6f9f7"
+ nickname="Frederik Vanrenterghem"
+ subject="comment 12"
+ date="2013-05-14T04:07:37Z"
+ content="""
+Thanks Joey, this worked. The symlinks are in the directory and I can get the actual file contents (except for one attempt that failed - I'll need to check why that didn't seem to have propagated to the bare repository.)
+"""]]

doc/forum/Does_migrate_ensure_data_integrity__63__/comment_2_f389b924c8531b35fdf5dedd10fc8000._comment

@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="GLITTAH"
+ ip="154.51.136.37"
+ subject="comment 2"
+ date="2013-05-14T15:54:08Z"
+ content="""
+An even better solution.  Thanks for your time and effort joey!
+"""]]

doc/install/Windows.mdwn

@@ -1,10 +1,26 @@
-git-annex has recently been ported to Windows.
+git-annex has recently been ported to Windows!
 
-This port is in an alpha state; many things will not work. See
+* First, [install git](http://git-scm.com/downloads)
-[[todo/windows_support]] for current status. Note especially that
+* Then, [install git-annex](http://downloads.kitenet.net/git-annex/windows/current/)
-git-annex always uses [[direct_mode]] on Windows.
+
+This port is in an early state. While it works well enough to use
+git-annex, many things will not work. See [[todo/windows_support]] for
+current status. Note especially that git-annex always uses [[direct_mode]]
+on Windows.
+
+## building it yourself
 
 To build git-annex from source on Windows, you need to install
-the Haskell Platform and Cygwin. When installing Cygwin, select
+the Haskell Platform and Cygwin. Use Cygwin to install gcc, rsync, git,
-to install gcc, make, rsync, and ssh. Then, within Cygwin, git-annex
+ssh, and gpg.
-can be compiled following the instructions for [[using cabal|cabal]].
+
+Then, within Cygwin, git-annex can be compiled following the instructions
+for [[using cabal|cabal]].
+
+Once git-annex is built, the NullSoft installer can be built, as follows:
+
+<pre>
+	cabal install ncis
+	ghc --make Build/NullSoftInstaller.hs
+	Build/NullSoftInstaller.exe
+</pre>

doc/install/Windows/comment_1_77e6b1023fee32277f1890def86b0106._comment

@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkGCmVc5qIJaQQgG82Hc5zzBdAVdhe2JEM"
+ nickname="Bruno"
+ subject="comment 1"
+ date="2013-05-15T18:29:19Z"
+ content="""
+Did you have any problem installing unix-2.6.0.1?
+
+I got :
+
+    cabal.exe: Missing dependencies on foreign libraries:
+    * Missing (or bad) header file: HsUnix.h
+    * Missing C libraries: rt, dl
+"""]]

doc/special_remotes/bup/comment_11_b53bceb0058acf4d1ab12ea4853ee443._comment

@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnAvbXOnK57sqgvZvxkbG74NUKBDwKDcuk"
+ nickname="Tim"
+ subject="bup location data not synced through annex assistant"
+ date="2013-05-15T15:08:54Z"
+ content="""
+I set up 2 servers running git annex assistant, both with a ~/annex dir and an additional ~/annex-bup bup repo. There is no additional cloud repository.
+For test, I added my /etc dir which uploaded correctly from server1, but which never arrived on server2
+
+    bup@bup1:~/annex/etc$ git annex whereis updatedb.conf
+    whereis updatedb.conf (3 copies) 
+  	    687d3a7f-4798-4dbe-8774-1785b8ab6b7d -- here (bup@bup1:~/annex)
+   	    adfc1307-771f-40e9-b794-bae2e1f21b8b -- bup2-annex-bup
+   	    e4e0ac0b-992a-4312-a4ac-fc8d3d9f7c0f -- bup1-annex-bup
+    ok
+
+    bup@bup2:~/annex/etc$ git annex whereis updatedb.conf
+    whereis updatedb.conf (1 copy) 
+  	    687d3a7f-4798-4dbe-8774-1785b8ab6b7d -- bup1 (bup@bup1:~/annex)
+    ok
+
+As you can see, server 2 just doesn't know the data is already on it's own disk in it's local bup repo.
+Is there a reason this data does not get synced? Should I set up a transfer repo?
+"""]]

doc/special_remotes/bup/comment_12_65d923226cf6120349d807c5c60f640c._comment

@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnAvbXOnK57sqgvZvxkbG74NUKBDwKDcuk"
+ nickname="Tim"
+ subject="my bad"
+ date="2013-05-15T15:39:31Z"
+ content="""
+Sorry, looks like I did initremote twice on the same folder, instead of enableremote the second time...
+"""]]

doc/todo/windows_support.mdwn

@@ -3,11 +3,10 @@ now! --[[Joey]]
 
 ## status
 
+* Does not support encryption with gpg.
 * Does not work with Cygwin's build of git (that git does not consistently
   support use of DOS style paths, which git-annex uses on Windows). 
   Must use the upstream build of git for Windows.
-* OTOH, you do need to install Cygwin's build of rsync (unless you can
-  find another one), and Cygwin's build of ssh.
 * test suite doesn't work
 * Bad file locking, it's probably not safe to run more than one git-annex
   process at the same time on Windows.

standalone/android/haskell-patches/iproute_1.2.11_0001-build-without-IPv6-stuff.patch

@@ -0,0 +1,47 @@
+From 7beec2e707d59f9573aa2dc7c57bd2a62f16b480 Mon Sep 17 00:00:00 2001
+From: Joey Hess <joey@kitenet.net>
+Date: Wed, 15 May 2013 19:06:03 -0400
+Subject: [PATCH] build without IPv6 stuff
+
+---
+ Data/IP.hs      |    2 +-
+ Data/IP/Addr.hs |    3 +++
+ 2 files changed, 4 insertions(+), 1 deletion(-)
+
+diff --git a/Data/IP.hs b/Data/IP.hs
+index cffef93..ea486c9 100644
+--- a/Data/IP.hs
++++ b/Data/IP.hs
+@@ -6,7 +6,7 @@ module Data.IP (
+   -- ** IP data
+     IP (..)
+   , IPv4, toIPv4, fromIPv4, fromHostAddress, toHostAddress
+-  , IPv6, toIPv6, fromIPv6, fromHostAddress6, toHostAddress6
++  , IPv6, toIPv6, fromIPv6 -- , fromHostAddress6, toHostAddress6
+   -- ** IP range data
+   , IPRange (..)
+   , AddrRange (addr, mask, mlen)
+diff --git a/Data/IP/Addr.hs b/Data/IP/Addr.hs
+index faaf0c7..5b556fb 100644
+--- a/Data/IP/Addr.hs
++++ b/Data/IP/Addr.hs
+@@ -312,6 +312,7 @@ toHostAddress (IP4 addr4)
+   | byteOrder == LittleEndian = fixByteOrder addr4
+   | otherwise                 = addr4
+ 
++{-
+ -- | The 'fromHostAddress6' function converts 'HostAddress6' to 'IPv6'.
+ fromHostAddress6 :: HostAddress6 -> IPv6
+ fromHostAddress6 = IP6
+@@ -320,6 +321,8 @@ fromHostAddress6 = IP6
+ toHostAddress6 :: IPv6 -> HostAddress6
+ toHostAddress6 (IP6 addr6) = addr6
+ 
++-}
++
+ fixByteOrder :: Word32 -> Word32
+ fixByteOrder s = d1 .|. d2 .|. d3 .|. d4
+   where
+-- 
+1.7.10.4
+

standalone/android/install-haskell-packages

@@ -157,6 +157,8 @@ install_pkgs () {
 	unpatched SHA-1.6.1
 	onlycross unpatched data-endian-0.0.1
 	unpatched hinotify-0.3.5
+	patched iproute 1.2.11
+	unpatched DNS
 	
 	cd ..
 	rm -rf tmp