From 2f52d8a28c8fb4d630b2c13803e6eaff231e051d Mon Sep 17 00:00:00 2001
From: "https://tribut.de/" <Felix@web>
Date: Sat, 12 Nov 2016 13:15:29 +0000
Subject: [PATCH 001/367]

---
 ...4___during___34__git_annex_sync__34__.mdwn | 52 +++++++++++++++++++
 1 file changed, 52 insertions(+)
 create mode 100644 doc/bugs/__34__commitBuffer__58___invalid_argument___40__invalid_character__41____34___during___34__git_annex_sync__34__.mdwn

diff --git a/doc/bugs/__34__commitBuffer__58___invalid_argument___40__invalid_character__41____34___during___34__git_annex_sync__34__.mdwn b/doc/bugs/__34__commitBuffer__58___invalid_argument___40__invalid_character__41____34___during___34__git_annex_sync__34__.mdwn
new file mode 100644
index 0000000000..d4e3ad4716
--- /dev/null
+++ b/doc/bugs/__34__commitBuffer__58___invalid_argument___40__invalid_character__41____34___during___34__git_annex_sync__34__.mdwn
@@ -0,0 +1,52 @@
+### Please describe the problem.
+
+In my unlocked adjusted branch, I get a lot of errors during "git annex sync". It appears to work fine otherwise (the files actually get synced). Below is what I see on the terminal. The repository is otherwise clean (no local or remote changes).
+This has started to happen around a month ago, though I cannot pinpoint the exact version. This is in the same repo you used to debug the disappearing files in direct mode recently (thanks a lot btw!).
+
+### What version of git-annex are you using? On what operating system?
+
+[[!format sh """
+$ git annex version
+git-annex version: 6.20161110-gd48f4ca
+build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+$ lsb_release -a
+No LSB modules are available.
+Distributor ID: Debian
+Description:    Debian GNU/Linux 8.6 (jessie)
+Release:        8.6
+Codename:       jessie
+"""]]
+
+### Please provide any additional information below.
+
+[[!format sh """
+$ git annex sync --content
+commit
+On branch adjusted/master(unlocked)
+nothing to commit, working tree clean
+ok
+pull origin
+remote: Counting objects: 113, done.
+remote: Compressing objects: 100% (113/113), done.
+remote: Total 113 (delta 112), reused 0 (delta 0)
+Receiving objects: 100% (113/113), 7.16 KiB | 0 bytes/s, done.
+Resolving deltas: 100% (112/112), completed with 112 local objects.
+From /srv/annex/bilder
+   97a4806..78cb4ef  git-annex  -> origin/git-annex
+ok
+(merging origin/git-annex into git-annex...)
+
+git-annex: fd:25: commitBuffer: invalid argument (invalid character)
+failed
+
+git-annex: fd:25: commitBuffer: invalid argument (invalid character)
+failed
+
+[...]
+
+git-annex: fd:25: commitBuffer: invalid argument (invalid character)
+failed
+git-annex: sync: 2653 failed
+"""]]

From 0718de20266dd3d997828bb55de678fd9b85f31c Mon Sep 17 00:00:00 2001
From: "mail@4e627fd997ef5ca9f75e62ffc0aba5b27bd6aea1" <mail@web>
Date: Sat, 12 Nov 2016 18:17:28 +0000
Subject: [PATCH 002/367]

---
 ...git_annex_get__96___files_left_broken.mdwn | 40 +++++++++++++++++++
 1 file changed, 40 insertions(+)
 create mode 100644 doc/bugs/When_stopping___96__git_annex_get__96___files_left_broken.mdwn

diff --git a/doc/bugs/When_stopping___96__git_annex_get__96___files_left_broken.mdwn b/doc/bugs/When_stopping___96__git_annex_get__96___files_left_broken.mdwn
new file mode 100644
index 0000000000..a8df723548
--- /dev/null
+++ b/doc/bugs/When_stopping___96__git_annex_get__96___files_left_broken.mdwn
@@ -0,0 +1,40 @@
+### Please describe the problem.
+
+```
+> git annex get Narnia/
+get Narnia/Course of a Generation/01 Sail Around the World.mp3 (from Seagate...) 
+SHA256E-s8395599--2fea961006a279f0765c45755b35a06f0a4fc6bfbab6118182ebc693d7b47a91.mp3
+      8,395,599 100%   29.65MB/s    0:00:00 (xfr#1, to-chk=0/1)
+(checksum...) ^C⏎                                   
+```
+
+```
+> mpv ~/Music/sorted/Narnia/Course\ of\ a\ Generation/
+Playing: /home/philip/Music/sorted/Narnia/Course of a Generation/
+[file] This is a directory - adding to playlist.
+
+Playing: /home/philip/Music/sorted/Narnia/Course of a Generation/01 Sail Around the World.mp3
+Failed to recognize file format.
+
+Playing: /home/philip/Music/sorted/Narnia/Course of a Generation/02 When the Stars Are Falling.mp3
+```
+
+```
+> git annex version
+git-annex version: 6.20161012
+build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+local repository version: 6
+supported repository versions: 3 5 6
+upgrade supported from repository versions: 0 1 2 3 4 5
+operating system: linux x86_64
+```
+
+Any consecutive `git annex get` commands don’t notice that the file is not completely transferred and leave it in a broken state.
+`git annex get --failed` does not correct the problem.
+
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Yes, it (kind of) works for keeping my music library in sync.

From 1e88c1257622c0b0bc8fe67b02718aba1974c05b Mon Sep 17 00:00:00 2001
From: "mail@de3e4e58ccb420b8d9b78ef262ee0c80e021f199" <mail@web>
Date: Sun, 13 Nov 2016 01:23:15 +0000
Subject: [PATCH 003/367]

---
 ...ile_name_causes_git_annex_add_to_fail.mdwn | 43 +++++++++++++++++++
 1 file changed, 43 insertions(+)
 create mode 100644 doc/bugs/Single_space_in_file_name_causes_git_annex_add_to_fail.mdwn

diff --git a/doc/bugs/Single_space_in_file_name_causes_git_annex_add_to_fail.mdwn b/doc/bugs/Single_space_in_file_name_causes_git_annex_add_to_fail.mdwn
new file mode 100644
index 0000000000..121488972f
--- /dev/null
+++ b/doc/bugs/Single_space_in_file_name_causes_git_annex_add_to_fail.mdwn
@@ -0,0 +1,43 @@
+### Please describe the problem.
+
+If a filename has a single space (and only one space), `git annex add` will fail out with the following message:
+
+    add one two git-annex: unknown response from git cat-file ("HEAD:./one two missing",Ref "HEAD:./one two")
+    CallStack (from HasCallStack):
+      error, called at ./Git/CatFile.hs:102:28 in main:Git.CatFile
+
+### What steps will reproduce the problem?
+
+Run the following:
+
+    git init .
+    git annex init
+    touch "one two"
+    # this will cause error
+    git annex add "one two"
+    touch "one two three"
+    # this is fine
+    git annex add "one two three"
+
+### What version of git-annex are you using? On what operating system?
+
+Output of `git annex version`
+
+    git-annex version: 6.20161027
+    build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+    remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+    local repository version: 5
+    supported repository versions: 3 5 6
+    upgrade supported from repository versions: 0 1 2 3 4 5
+    operating system: linux x86_64
+
+Operating System: Linux (NixOS 16.09.909.238c7e0 (Flounder))
+
+### Please provide any additional information below.
+
+Maybe related to [https://git-annex.branchable.com/forum/unknown_response_from_git_cat-file/](https://git-annex.branchable.com/forum/unknown_response_from_git_cat-file/) or [https://git-annex.branchable.com/bugs/git_annex_import_fails_on_filenames_with_newlines_in_them/](https://git-annex.branchable.com/bugs/git_annex_import_fails_on_filenames_with_newlines_in_them/)?
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Just starting out with it as a way of archiving some of my media seems to work great apart from this. Thanks a bunch!

From b634e006c46c654adbc752f20119a6c63cfd61b1 Mon Sep 17 00:00:00 2001
From: "mail@de3e4e58ccb420b8d9b78ef262ee0c80e021f199" <mail@web>
Date: Sun, 13 Nov 2016 02:40:02 +0000
Subject: [PATCH 004/367] Added additional comment about it working if I build
 from source

---
 ...pace_in_file_name_causes_git_annex_add_to_fail.mdwn | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/doc/bugs/Single_space_in_file_name_causes_git_annex_add_to_fail.mdwn b/doc/bugs/Single_space_in_file_name_causes_git_annex_add_to_fail.mdwn
index 121488972f..14f772c5fd 100644
--- a/doc/bugs/Single_space_in_file_name_causes_git_annex_add_to_fail.mdwn
+++ b/doc/bugs/Single_space_in_file_name_causes_git_annex_add_to_fail.mdwn
@@ -38,6 +38,16 @@ Operating System: Linux (NixOS 16.09.909.238c7e0 (Flounder))
 
 Maybe related to [https://git-annex.branchable.com/forum/unknown_response_from_git_cat-file/](https://git-annex.branchable.com/forum/unknown_response_from_git_cat-file/) or [https://git-annex.branchable.com/bugs/git_annex_import_fails_on_filenames_with_newlines_in_them/](https://git-annex.branchable.com/bugs/git_annex_import_fails_on_filenames_with_newlines_in_them/)?
 
+EDIT: Somewhat surprisingly, if I build from source using `cabal`, everything works fine.
+
+    .cabal-sandbox/bin/git-annex version
+    git-annex version: 6.20161113-g1e88c12
+    build flags: Assistant Webapp Pairing Testsuite WebDAV Inotify ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+    remote types: git gcrypt bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+
+Not sure whether this means that this bug is actually fixed or whether it's an artifact of how things are built in Nix.
+
 ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
 
 Just starting out with it as a way of archiving some of my media seems to work great apart from this. Thanks a bunch!

From 2e57f97a67a9526e8caa29c51cc8f55863daddb0 Mon Sep 17 00:00:00 2001
From: "neocryptek@659edac901ffbc8e541a974f8f18987eeafc63bd" <neocryptek@web>
Date: Sun, 13 Nov 2016 22:39:44 +0000
Subject: [PATCH 005/367] Added a comment

---
 ...omment_3_9859c46db3527ad329c8e0df06edd153._comment | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 doc/forum/Sending_requests_across_the_network/comment_3_9859c46db3527ad329c8e0df06edd153._comment

diff --git a/doc/forum/Sending_requests_across_the_network/comment_3_9859c46db3527ad329c8e0df06edd153._comment b/doc/forum/Sending_requests_across_the_network/comment_3_9859c46db3527ad329c8e0df06edd153._comment
new file mode 100644
index 0000000000..c63404e0e8
--- /dev/null
+++ b/doc/forum/Sending_requests_across_the_network/comment_3_9859c46db3527ad329c8e0df06edd153._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="neocryptek@659edac901ffbc8e541a974f8f18987eeafc63bd"
+ nickname="neocryptek"
+ avatar="http://cdn.libravatar.org/avatar/d9bfdefa9b503f1ac4844a686618374e"
+ subject="comment 3"
+ date="2016-11-13T22:39:44Z"
+ content="""
+Thanks, that makes sense.
+
+All git annex repositories using the same branch will have the same (symlink) working directory right (assuming entire network has been synced eventually)?
+"""]]

From 7876298ee293547ca4c9cab7c20865696e7b5d6c Mon Sep 17 00:00:00 2001
From: "neocryptek@659edac901ffbc8e541a974f8f18987eeafc63bd" <neocryptek@web>
Date: Mon, 14 Nov 2016 00:46:04 +0000
Subject: [PATCH 006/367]

---
 ...illa_git_repo_as_special_remote__63__.mdwn | 27 +++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 doc/forum/vanilla_git_repo_as_special_remote__63__.mdwn

diff --git a/doc/forum/vanilla_git_repo_as_special_remote__63__.mdwn b/doc/forum/vanilla_git_repo_as_special_remote__63__.mdwn
new file mode 100644
index 0000000000..94fa548653
--- /dev/null
+++ b/doc/forum/vanilla_git_repo_as_special_remote__63__.mdwn
@@ -0,0 +1,27 @@
+Right now I have separate "normal" Git repositories and separate Git annex repositories and I would love to have Git annex track and sync everything for me. The problem I have is I'd like to use "real" Git content tracking for some data (ex: text files) where I'd like to get normal Git features with (ex: diff). I'd like to combine normal Git content tracking with Git annex location tracking and syncing if possible. Ideally the cost (ex: increased git repo size and git slowdown) of content tracking would not need to be propagated across the entire git annex network, just on repos that want it (just like git annex only copies content to clients who want it and symlink the rest).
+
+The largefiles config provides a mechanism to add content to git directly in git annex, but that cost would be applied across the entire network, not opt-in per client.
+
+Ideally I'd like this situation:
+
+1. Git annex tracking everything as symlinks. No content is checked into these git repos.
+2. A subset of git annex content (ex: subfolder) synced to a normal remote non-annex git repository (ex: GitHub). This Git repo has content tracked in git itself.
+
+And I could use the git annex repos to sync everything. Somehow the git annex repo would know that the #2 remote was a "special content git remote" and push any content updates as normal git content commits.
+
+Or an adjusted branch that had the content tracked and I could sync that content branch around to only the remotes where I wanted the content history stored in git (since adjusted branches don't seem to annex sync by default). But master would just track the symlinks of those files and be synced around to all annexes.
+
+Can adjusted branches do this somehow?
+
+Some references:
+
+* [[special_remotes/external]]
+* [[design/adjusted_branches]]
+* [[todo/hide_missing_files]]
+* [[tips/largefiles]]
+* [[submodules]]
+* [[forum/git-subtree_support__63__]]
+
+Thanks!
+
+-neocryptek

From c20505f02a9f0f6aec5e25e7d7a7667a98f510fa Mon Sep 17 00:00:00 2001
From: "gfa@1e86118cd41fbfea50004af221471ad97b55af18" <gfa@web>
Date: Mon, 14 Nov 2016 09:13:20 +0000
Subject: [PATCH 007/367] Added a comment: same on Debian

---
 ...mment_1_0cf0856c6408c9c588133023a3a6ba8f._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/bugs/Single_space_in_file_name_causes_git_annex_add_to_fail/comment_1_0cf0856c6408c9c588133023a3a6ba8f._comment

diff --git a/doc/bugs/Single_space_in_file_name_causes_git_annex_add_to_fail/comment_1_0cf0856c6408c9c588133023a3a6ba8f._comment b/doc/bugs/Single_space_in_file_name_causes_git_annex_add_to_fail/comment_1_0cf0856c6408c9c588133023a3a6ba8f._comment
new file mode 100644
index 0000000000..5f8b120e20
--- /dev/null
+++ b/doc/bugs/Single_space_in_file_name_causes_git_annex_add_to_fail/comment_1_0cf0856c6408c9c588133023a3a6ba8f._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="gfa@1e86118cd41fbfea50004af221471ad97b55af18"
+ nickname="gfa"
+ avatar="http://cdn.libravatar.org/avatar/4678da4da55c67fa668e31ea0a76b201"
+ subject="same on Debian"
+ date="2016-11-14T09:13:19Z"
+ content="""
+I face the same issue on Debian testing
+
+git-annex 6.20161012-1
+git 1:2.10.2-1
+"""]]

From 07ad19f4212293e0227fa131a11531f2099d1ec5 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 14 Nov 2016 13:26:34 -0400
Subject: [PATCH 008/367] git-annex enable-tor command

Tor unfortunately does not come out of the box configured to let hidden
services register themselves on the fly via the ControlPort.

And, changing the config to enable the ControlPort and a particular type
of auth for it may break something already using the ControlPort, or
lessen the security of the system.

So, this leaves only one option to us: Add a hidden service to the
torrc. git-annex enable-tor does so, and picks an unused high port for
tor to listen on for connections to the hidden service.

It's up to the caller to somehow pick a local port to listen on
that won't be used by something else. That may be difficult to do..

This commit was sponsored by Jochen Bartl on Patreon.
---
 CmdLine/GitAnnex.hs           |  2 +
 Command/EnableTor.hs          | 28 ++++++++++++++
 Utility/Tor.hs                | 71 +++++++++++++++++++++++++++++++++++
 doc/git-annex-enable-tor.mdwn | 25 ++++++++++++
 git-annex.cabal               |  1 +
 5 files changed, 127 insertions(+)
 create mode 100644 Command/EnableTor.hs
 create mode 100644 Utility/Tor.hs
 create mode 100644 doc/git-annex-enable-tor.mdwn

diff --git a/CmdLine/GitAnnex.hs b/CmdLine/GitAnnex.hs
index e989f3f438..0049ecb3c3 100644
--- a/CmdLine/GitAnnex.hs
+++ b/CmdLine/GitAnnex.hs
@@ -52,6 +52,7 @@ import qualified Command.Init
 import qualified Command.Describe
 import qualified Command.InitRemote
 import qualified Command.EnableRemote
+import qualified Command.EnableTor
 import qualified Command.Expire
 import qualified Command.Repair
 import qualified Command.Unused
@@ -142,6 +143,7 @@ cmds testoptparser testrunner =
 	, Command.Describe.cmd
 	, Command.InitRemote.cmd
 	, Command.EnableRemote.cmd
+	, Command.EnableTor.cmd
 	, Command.Reinject.cmd
 	, Command.Unannex.cmd
 	, Command.Uninit.cmd
diff --git a/Command/EnableTor.hs b/Command/EnableTor.hs
new file mode 100644
index 0000000000..8d9dd6f0aa
--- /dev/null
+++ b/Command/EnableTor.hs
@@ -0,0 +1,28 @@
+{- git-annex command
+ -
+ - Copyright 2016 Joey Hess <id@joeyh.name>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.EnableTor where
+
+import Command
+import Utility.Tor
+
+cmd :: Command
+cmd = noCommit $ dontCheck repoExists $
+	command "enable-tor" SectionPlumbing ""
+		paramNumber (withParams seek)
+
+seek :: CmdParams -> CommandSeek
+seek = withWords start
+
+start :: CmdParams -> CommandStart
+start (localport:[]) = case readish localport of
+	Nothing -> error "Bad localport"
+	Just lp -> do
+		(onionaddr, onionport) <- liftIO $ addHiddenService lp
+		liftIO $ putStrLn (onionaddr ++ ":" ++ show onionport)
+		stop
+start _ = error "Need 1 localport parameter"
diff --git a/Utility/Tor.hs b/Utility/Tor.hs
new file mode 100644
index 0000000000..b15a23dccc
--- /dev/null
+++ b/Utility/Tor.hs
@@ -0,0 +1,71 @@
+{- tor interface
+ -
+ - Copyright 2016 Joey Hess <id@joeyh.name>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Tor where
+
+import Common
+import Utility.ThreadScheduler
+import Data.Char
+
+type LocalPort = Int
+type OnionPort = Int
+type OnionAddress = String
+
+-- | Adds a hidden service connecting to localhost on the specified local port.
+-- This will only work if run as root, and tor has to already be running.
+--
+-- Picks a port number for the hidden service that is not used by any
+-- other hidden service (and is >= 1024). Returns the hidden service's
+-- onion address and port.
+
+-- If there is already a hidden service for the specified local port,
+-- returns its information without making any changes.
+addHiddenService :: LocalPort -> IO (OnionAddress, OnionPort)
+addHiddenService localport = do
+	ls <- map (separate isSpace) . lines <$> readFile torrc
+	let usedports = mapMaybe readish $
+		map (drop 1 . dropWhile (/= ':')) $
+		map snd $
+		filter (\(k, _) -> k == "HiddenServicePort") ls
+	let newport = Prelude.head $ filter (`notElem` usedports) [1024..]
+	let dir = libDir </> "hidden_service" ++ show localport
+	if localport `elem` usedports
+		then waithiddenservice 1 dir newport
+		else do
+			writeFile torrc $ unlines $
+				map (\(k, v) -> k ++ " " ++ v) ls ++
+				[ ""
+				, "HiddenServiceDir " ++ dir
+				, "HiddenServicePort " ++ show newport ++ 
+					" 127.0.0.1:" ++ show localport
+				]
+			-- Reload tor, so it will see the new hidden
+			-- service and generate the hostname file for it.
+			reloaded <- anyM (uncurry boolSystem)
+				[ ("systemctl", [Param "reload", Param "tor"])
+				, ("sefvice", [Param "tor", Param "reload"])
+				]
+			unless reloaded $
+				error "failed to reload tor, perhaps the tor service is not running"
+			waithiddenservice 120 dir newport
+  where
+	waithiddenservice :: Int -> FilePath -> OnionPort -> IO (OnionAddress, OnionPort)
+	waithiddenservice 0 _ _ = error "tor failed to create hidden service, perhaps the tor service is not running"
+	waithiddenservice n dir newport = do
+		v <- tryIO $ readFile (dir </> "hostname")
+		case v of
+			Right s | ".onion\n" `isSuffixOf` s -> 
+				return (takeWhile (/= '\n') s, newport)
+			_ -> do
+				threadDelaySeconds (Seconds 1)
+				waithiddenservice (n-1) dir newport
+
+torrc :: FilePath
+torrc = "/etc/tor/torrc"
+
+libDir :: FilePath
+libDir = "/var/lib/tor"
diff --git a/doc/git-annex-enable-tor.mdwn b/doc/git-annex-enable-tor.mdwn
new file mode 100644
index 0000000000..961eef1722
--- /dev/null
+++ b/doc/git-annex-enable-tor.mdwn
@@ -0,0 +1,25 @@
+# NAME
+
+git-annex enable-tor - enable tor hidden service
+
+# SYNOPSIS
+
+git annex enable-tor localport
+
+# DESCRIPTION
+
+This plumbing-level command enables a tor hidden service for git-annex,
+using the specified local port number. It outputs to stdout a line
+of the form "address.onion:onionport"
+
+This command has to be run by root, since it modifies `/etc/tor/torrc`.
+
+# SEE ALSO
+
+[[git-annex]](1)
+
+# AUTHOR
+
+Joey Hess <id@joeyh.name>
+
+Warning: Automatically converted into a man page by mdwn2man. Edit with care.
diff --git a/git-annex.cabal b/git-annex.cabal
index 65abc8d32d..dea5eb700b 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1063,6 +1063,7 @@ Executable git-annex
     Utility.ThreadLock
     Utility.ThreadScheduler
     Utility.Tmp
+    Utility.Tor
     Utility.Touch
     Utility.Url
     Utility.UserInfo

From 61fe128bbc09d1c1ea67a111b4836ee3b2fed5db Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 14 Nov 2016 15:04:16 -0400
Subject: [PATCH 009/367] no-xmpp branch

---
 doc/todo/xmpp_removal.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/todo/xmpp_removal.mdwn b/doc/todo/xmpp_removal.mdwn
index 9eb0407804..c517c33f98 100644
--- a/doc/todo/xmpp_removal.mdwn
+++ b/doc/todo/xmpp_removal.mdwn
@@ -21,5 +21,7 @@ telehash. But, can't wait on that forever..
 XMPP support is already disabled by default in some builds of git-annex,
 notably the stack build. It's never worked on Windows.
 
+The [[no-xmpp]] branch is ready for merging.
+
 Next step is probably to default the flag to false by default,
 except for in a few builds like the Debian package and standalone builds.

From 57d33f79238599014e2eeef569d61fc3021ee476 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 14 Nov 2016 16:35:45 -0400
Subject: [PATCH 010/367] use socket for tor hidden service

This avoids needing to bind to the right port before something else
does.

The socket is in /var/run/user/$uid/ which ought to be writable by only
that uid. At least it is on linux systems using systemd.

For Windows, may need to revisit this and use ports or something.

The first version of tor to support sockets for hidden services
was 0.2.6.3. That is not in Debian stable, but is available in
backports.

This commit was sponsored by andrea rota.
---
 Command/EnableTor.hs          | 20 +++++++----
 Utility/Tor.hs                | 67 ++++++++++++++++++++---------------
 doc/git-annex-enable-tor.mdwn |  6 ++--
 3 files changed, 55 insertions(+), 38 deletions(-)

diff --git a/Command/EnableTor.hs b/Command/EnableTor.hs
index 8d9dd6f0aa..1a54c6c5d8 100644
--- a/Command/EnableTor.hs
+++ b/Command/EnableTor.hs
@@ -10,19 +10,25 @@ module Command.EnableTor where
 import Command
 import Utility.Tor
 
+-- This runs as root, so avoid making any commits or initializing
+-- git-annex, as that would create root-owned files.
 cmd :: Command
 cmd = noCommit $ dontCheck repoExists $
 	command "enable-tor" SectionPlumbing ""
-		paramNumber (withParams seek)
+		"userid uuid" (withParams seek)
 
 seek :: CmdParams -> CommandSeek
 seek = withWords start
 
 start :: CmdParams -> CommandStart
-start (localport:[]) = case readish localport of
-	Nothing -> error "Bad localport"
-	Just lp -> do
-		(onionaddr, onionport) <- liftIO $ addHiddenService lp
-		liftIO $ putStrLn (onionaddr ++ ":" ++ show onionport)
+start (suserid:uuid:[]) = case readish suserid of
+	Nothing -> error "Bad userid"
+	Just userid -> do
+		(onionaddr, onionport, onionsocket) <- liftIO $
+			addHiddenService userid uuid
+		liftIO $ putStrLn $
+			onionaddr ++ ":" ++ 
+			show onionport ++ " " ++
+			show onionsocket
 		stop
-start _ = error "Need 1 localport parameter"
+start _ = error "Bad params"
diff --git a/Utility/Tor.hs b/Utility/Tor.hs
index b15a23dccc..a0a6090089 100644
--- a/Utility/Tor.hs
+++ b/Utility/Tor.hs
@@ -9,39 +9,39 @@ module Utility.Tor where
 
 import Common
 import Utility.ThreadScheduler
+import System.PosixCompat.Types
 import Data.Char
 
-type LocalPort = Int
 type OnionPort = Int
 type OnionAddress = String
+type OnionSocket = FilePath
 
--- | Adds a hidden service connecting to localhost on the specified local port.
+-- | Adds a hidden service connecting to localhost, using some kind
+-- of unique identifier.
+--
 -- This will only work if run as root, and tor has to already be running.
 --
 -- Picks a port number for the hidden service that is not used by any
 -- other hidden service (and is >= 1024). Returns the hidden service's
--- onion address and port.
-
--- If there is already a hidden service for the specified local port,
--- returns its information without making any changes.
-addHiddenService :: LocalPort -> IO (OnionAddress, OnionPort)
-addHiddenService localport = do
-	ls <- map (separate isSpace) . lines <$> readFile torrc
-	let usedports = mapMaybe readish $
-		map (drop 1 . dropWhile (/= ':')) $
-		map snd $
-		filter (\(k, _) -> k == "HiddenServicePort") ls
-	let newport = Prelude.head $ filter (`notElem` usedports) [1024..]
-	let dir = libDir </> "hidden_service" ++ show localport
-	if localport `elem` usedports
-		then waithiddenservice 1 dir newport
-		else do
+-- onion address, port, and the unix socket file to use.
+-- 
+-- If there is already a hidden service for the specified unique
+-- identifier, returns its information without making any changes.
+addHiddenService :: UserID -> String -> IO (OnionAddress, OnionPort, OnionSocket)
+addHiddenService uid ident = do
+	ls <- lines <$> readFile torrc
+	let portssocks = mapMaybe (parseportsock . separate isSpace) ls
+	case filter (\(_, s) -> s == sockfile) portssocks of
+		((p, _s):_) -> waithiddenservice 1 p
+		_ -> do
+			let newport = Prelude.head $
+				filter (`notElem` map fst portssocks) [1024..]
 			writeFile torrc $ unlines $
-				map (\(k, v) -> k ++ " " ++ v) ls ++
+				ls ++
 				[ ""
-				, "HiddenServiceDir " ++ dir
+				, "HiddenServiceDir " ++ hsdir
 				, "HiddenServicePort " ++ show newport ++ 
-					" 127.0.0.1:" ++ show localport
+					" unix:" ++ sockfile
 				]
 			-- Reload tor, so it will see the new hidden
 			-- service and generate the hostname file for it.
@@ -51,21 +51,32 @@ addHiddenService localport = do
 				]
 			unless reloaded $
 				error "failed to reload tor, perhaps the tor service is not running"
-			waithiddenservice 120 dir newport
+			waithiddenservice 120 newport
   where
-	waithiddenservice :: Int -> FilePath -> OnionPort -> IO (OnionAddress, OnionPort)
-	waithiddenservice 0 _ _ = error "tor failed to create hidden service, perhaps the tor service is not running"
-	waithiddenservice n dir newport = do
-		v <- tryIO $ readFile (dir </> "hostname")
+	parseportsock ("HiddenServicePort", l) = do
+		p <- readish $ takeWhile (not . isSpace) l
+		return (p, drop 1 (dropWhile (/= ':') l))
+	parseportsock _ = Nothing
+	
+	hsdir = libDir </> "hidden_service_" ++ show uid ++ "_" ++ ident
+	sockfile = runDir uid </> ident ++ ".sock"
+
+	waithiddenservice :: Int -> OnionPort -> IO (OnionAddress, OnionPort, OnionSocket)
+	waithiddenservice 0 _ = error "tor failed to create hidden service, perhaps the tor service is not running"
+	waithiddenservice n p = do
+		v <- tryIO $ readFile (hsdir </> "hostname")
 		case v of
 			Right s | ".onion\n" `isSuffixOf` s -> 
-				return (takeWhile (/= '\n') s, newport)
+				return (takeWhile (/= '\n') s, p, sockfile)
 			_ -> do
 				threadDelaySeconds (Seconds 1)
-				waithiddenservice (n-1) dir newport
+				waithiddenservice (n-1) p
 
 torrc :: FilePath
 torrc = "/etc/tor/torrc"
 
 libDir :: FilePath
 libDir = "/var/lib/tor"
+
+runDir :: UserID -> FilePath
+runDir uid = "/var/run/user" </> show uid
diff --git a/doc/git-annex-enable-tor.mdwn b/doc/git-annex-enable-tor.mdwn
index 961eef1722..b44cf817cf 100644
--- a/doc/git-annex-enable-tor.mdwn
+++ b/doc/git-annex-enable-tor.mdwn
@@ -4,13 +4,13 @@ git-annex enable-tor - enable tor hidden service
 
 # SYNOPSIS
 
-git annex enable-tor localport
+git annex enable-tor userid uuid
 
 # DESCRIPTION
 
 This plumbing-level command enables a tor hidden service for git-annex,
-using the specified local port number. It outputs to stdout a line
-of the form "address.onion:onionport"
+using the specified repository uuid and userid. 
+It outputs to stdout a line of the form "address.onion:onionport socketfile"
 
 This command has to be run by root, since it modifies `/etc/tor/torrc`.
 

From 35116c5d044177f82d4aef1f795a0ef9e78ec707 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 14 Nov 2016 16:49:18 -0400
Subject: [PATCH 011/367] devblog

---
 doc/devblog/day_425__tor.mdwn | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 doc/devblog/day_425__tor.mdwn

diff --git a/doc/devblog/day_425__tor.mdwn b/doc/devblog/day_425__tor.mdwn
new file mode 100644
index 0000000000..08fe21cdd5
--- /dev/null
+++ b/doc/devblog/day_425__tor.mdwn
@@ -0,0 +1,23 @@
+Have waited too long for some next-generation encrypted P2P network, like
+telehash to emerge. Time to stop waiting; tor hidden services are not as
+cutting edge, but should work. Updated the [[design|design/assistant/telehash]]
+and started implementation in the `tor` branch.
+
+Unfortunately, Tor's default configuration does not enable the ControlPort.
+And, changing that in the configuration could be problimatic. This
+makes it harder than it ought to be to register a tor hidden service.
+So, I implemented a `git annex enable-tor` command, which can be run as root
+to set it up. The webapp will probably use `su-to-root` or `gksu` to run it. 
+There's some Linux-specific parts in there, and it uses a socket for
+communication between tor and the hidden service, which may cause problems
+for Windows porting later.
+
+Next step will be to get `git annex remotedaemon` to run as a tor hidden
+service.
+
+Also made a `no-xmpp` branch which removes xmpp support from the assistant.
+That will remove 3000 lines of code when it's merged. Will probably wait
+until after tor hidden services are working.
+
+Today's work was sponsored by Jake Vosloo on
+[Patreon](https://www.patreon.com/joeyh/).

From 67c1e87f0568d0d8f2ef722461c3d4ab03bdded8 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 14 Nov 2016 18:37:56 -0400
Subject: [PATCH 012/367] local lan detection

---
 doc/design/assistant/telehash.mdwn | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/doc/design/assistant/telehash.mdwn b/doc/design/assistant/telehash.mdwn
index 373f1a5758..5c410999fe 100644
--- a/doc/design/assistant/telehash.mdwn
+++ b/doc/design/assistant/telehash.mdwn
@@ -123,6 +123,14 @@ so won't want to type that in. Need discovery.
   for Bob to confirm he's ready to finish pairing, this will fail,
   because Bob won't get to that point if the authtoken is intercepted.
 
+## local lan detection
+
+At connection time, after authentication, the remote can send
+(ip address, ssh host key). Try sshing to the ip address to check if
+the host key matches. If so, can enable a ssh remote, which will
+be cheaper than using the transport. Send the ssh public key back to the
+remote to get it authorized.
+
 ## remotedaemon
 
 See [[git-remote-daemon]] for its design.

From 4805b6e31d313a7a3f19594c6e0e1066aef08e18 Mon Sep 17 00:00:00 2001
From: thowz <thowz@web>
Date: Tue, 15 Nov 2016 02:27:52 +0000
Subject: [PATCH 013/367]

---
 ..._doesn__39__t_work_for_special_remote.mdwn | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 doc/bugs/git-annex-fsck___34__-all__34___flag_doesn__39__t_work_for_special_remote.mdwn

diff --git a/doc/bugs/git-annex-fsck___34__-all__34___flag_doesn__39__t_work_for_special_remote.mdwn b/doc/bugs/git-annex-fsck___34__-all__34___flag_doesn__39__t_work_for_special_remote.mdwn
new file mode 100644
index 0000000000..b9895056fc
--- /dev/null
+++ b/doc/bugs/git-annex-fsck___34__-all__34___flag_doesn__39__t_work_for_special_remote.mdwn
@@ -0,0 +1,26 @@
+### Please describe the problem.
+I tried to use `git-annex-fsck --all --from remote` to check files on a special remote, but git-annex did a scan of the local repo instead. If I don't use the `--all` flag, it correctly checks the files on the remote (but just the files in the current checked out branch).
+
+### What steps will reproduce the problem?
+    mkdir repo
+    mkdir special
+    cd repo
+    git init
+    git annex init
+    git annex initremote special type=directory directory=../special encryption=none
+    touch testfile
+    git annex add testfile
+    git annex copy testfile --to special
+    chmod -R +w ../special/*
+    rm -r ../special/*
+    git annex fsck --all --from special   # should check special remote but checks local repo instead
+    git diff git-annex^ git-annex         # activity log shows that it checked special remote
+    git annex fsck --from special         # correctly checks special remote, identifies missing file
+
+
+### What version of git-annex are you using? On what operating system?
+6.20161012 on Ubuntu 16.10
+
+### Have you had any luck using git-annex before?
+Yes, it's been very helpful for managing large files between laptops, desktops, external storage, and remote storage.
+

From eaf86f4ff51d9201f0cb902ed14ad837b6130933 Mon Sep 17 00:00:00 2001
From: "https://launchpad.net/~stephane-gourichon-lpad"
 <stephane-gourichon-lpad@web>
Date: Tue, 15 Nov 2016 10:58:32 +0000
Subject: [PATCH 014/367] Added a comment: "Hmm, guyz? Are you serious with
 these scripts?" Well, what's the matter?

---
 ..._603db6818d33663b70b917c04fd8485b._comment | 30 +++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_7_603db6818d33663b70b917c04fd8485b._comment

diff --git a/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_7_603db6818d33663b70b917c04fd8485b._comment b/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_7_603db6818d33663b70b917c04fd8485b._comment
new file mode 100644
index 0000000000..5527c2b430
--- /dev/null
+++ b/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_7_603db6818d33663b70b917c04fd8485b._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~stephane-gourichon-lpad"
+ nickname="stephane-gourichon-lpad"
+ avatar="http://cdn.libravatar.org/avatar/02d4a0af59175f9123720b4481d55a769ba954e20f6dd9b2792217d9fa0c6089"
+ subject="&quot;Hmm, guyz? Are you serious with these scripts?&quot; Well, what's the matter?"
+ date="2016-11-15T10:58:32Z"
+ content="""
+## Wow, scary
+
+Dilyin's comment is scary.  It suggests bad things can happen, but is not very clear.
+
+Bloated history is one thing.  
+Obviously broken repo is bad but can be (slowly) recovered from remotes.  
+Subtly crippled history that you don't notice can be a major problem (especially once you have propagated it to all your remotes to \"recover from bloat\").
+
+## More common than it seems
+
+There's a case probably more common than people actually report: mistakenly doing `git add` instead of `git annex add` and realizing it only after a number of commits. Doing `git annex add` at that time will have the file duplicated (regular git and annex). 
+
+Extra wish: when doing `git annex add` of a file that is already present in git history, `git-annex` could notice and tell.
+
+## Simple solution?
+
+Can anyone elaborate on the scripts provided here, are they safe?  What can happen if improperly used or in corner cases?
+
+* \"files are replaced with symlinks and are in the index\" -> so what ?
+* \"Make sure that you don't have annex.largefiles settings that would prevent annexing the files.\" -> What would happen? Also `.gitattributes`.
+
+Thank you.
+"""]]

From 23a8134e0d9f398635327aa54ef3c4de6169c3a9 Mon Sep 17 00:00:00 2001
From: "christopher@5845ecd3cef9edadd4dc084df00e1fa60ce311eb" <christopher@web>
Date: Tue, 15 Nov 2016 12:15:37 +0000
Subject: [PATCH 015/367] Added a comment

---
 ..._54bd11140dbe794182263c1a062ad031._comment | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 doc/bugs/Webapp_missing_CSS_and_JS_resources___40__401_Unauthorized__41__/comment_3_54bd11140dbe794182263c1a062ad031._comment

diff --git a/doc/bugs/Webapp_missing_CSS_and_JS_resources___40__401_Unauthorized__41__/comment_3_54bd11140dbe794182263c1a062ad031._comment b/doc/bugs/Webapp_missing_CSS_and_JS_resources___40__401_Unauthorized__41__/comment_3_54bd11140dbe794182263c1a062ad031._comment
new file mode 100644
index 0000000000..cb6e9b4d22
--- /dev/null
+++ b/doc/bugs/Webapp_missing_CSS_and_JS_resources___40__401_Unauthorized__41__/comment_3_54bd11140dbe794182263c1a062ad031._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="christopher@5845ecd3cef9edadd4dc084df00e1fa60ce311eb"
+ nickname="christopher"
+ avatar="http://cdn.libravatar.org/avatar/4b722efb21f38d9944730c93727bc602"
+ subject="comment 3"
+ date="2016-11-15T12:15:37Z"
+ content="""
+Hi Joey,
+
+I installed git-annex using the homebrew recipe from https://github.com/Homebrew/homebrew-core/blob/master/Formula/git-annex.rb, OS X 10.11.6 (15G31)
+
+These are the dependencies reported by homebrew:
+
+Build: ghc ✔, cabal-install ✔, pkg-config ✔
+Required: gsasl ✔, libidn ✔, libmagic ✔, gnutls ✔, quvi ✔
+
+I've re-installed using \"brew install git-annex --HEAD\" to pull in your latest commit and I can confirm that everything works as expected and the /static/ resources load correctly.
+
+Thanks,
+Chris
+"""]]

From aa0da3663a26b408604126ddf13583f84ab250e8 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 15 Nov 2016 11:02:21 -0400
Subject: [PATCH 016/367] close

---
 ...sing_CSS_and_JS_resources___40__401_Unauthorized__41__.mdwn | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/Webapp_missing_CSS_and_JS_resources___40__401_Unauthorized__41__.mdwn b/doc/bugs/Webapp_missing_CSS_and_JS_resources___40__401_Unauthorized__41__.mdwn
index 0c3fc16a1a..17fe3ac254 100644
--- a/doc/bugs/Webapp_missing_CSS_and_JS_resources___40__401_Unauthorized__41__.mdwn
+++ b/doc/bugs/Webapp_missing_CSS_and_JS_resources___40__401_Unauthorized__41__.mdwn
@@ -30,4 +30,5 @@ operating system: darwin x86_64
 
 ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
 
-
+> [[done]], my fix worked! Don't know entirely why it was needed..
+> --[[Joey]]

From 6416ae9c09f54c062c05cc686ade35c2e08c1434 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 15 Nov 2016 11:19:57 -0400
Subject: [PATCH 017/367] unbreak all the autobuilders

git-annex.cabal: Loosen bounds on persistent to allow 2.5, which on Debian
has been patched to work with esqueleto. This may break cabal's resolver on
non-Debian systems; if so, either use stack to build, or run cabal with
--constraint='persistent ==2.2.4.1' Hopefully this mess with esqueleto will
be resolved soon.

https://github.com/prowdsponsor/esqueleto/issues/137
---
 CHANGELOG       | 11 +++++++++++
 git-annex.cabal |  5 +----
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index 273bf520e8..33f434df46 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,14 @@
+git-annex (6.20161112) UNRELEASED; urgency=medium
+
+  * git-annex.cabal: Loosen bounds on persistent to allow 2.5, which
+    on Debian has been patched to work with esqueleto.
+    This may break cabal's resolver on non-Debian systems;
+    if so, either use stack to build, or run cabal with 
+    --constraint='persistent ==2.2.4.1'
+    Hopefully this mess with esqueleto will be resolved soon.
+
+ -- Joey Hess <id@joeyh.name>  Tue, 15 Nov 2016 11:15:27 -0400
+
 git-annex (6.20161111) unstable; urgency=medium
 
   * Restarting a crashing git process could result in filename encoding
diff --git a/git-annex.cabal b/git-annex.cabal
index 65abc8d32d..46b08d22d0 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -359,10 +359,7 @@ Executable git-annex
    old-locale,
    esqueleto,
    persistent-sqlite, 
-   -- Old version needed due to
-   -- https://github.com/prowdsponsor/esqueleto/issues/137
-   -- and also temporarily to make ghc 8 builds work
-   persistent (< 2.5),
+   persistent,
    persistent-template,
    aeson,
    unordered-containers,

From 501c29a2ad846a2e08b184c047501ba4a4db3da6 Mon Sep 17 00:00:00 2001
From: "grawity@2ea26be48562f66fcb9b66307da72b1e2e37453f" <grawity@web>
Date: Tue, 15 Nov 2016 18:01:18 +0000
Subject: [PATCH 018/367] Added a comment

---
 ...omment_1_1dd41fa32eb3867d764f3238005b5b81._comment | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 doc/devblog/day_425__tor/comment_1_1dd41fa32eb3867d764f3238005b5b81._comment

diff --git a/doc/devblog/day_425__tor/comment_1_1dd41fa32eb3867d764f3238005b5b81._comment b/doc/devblog/day_425__tor/comment_1_1dd41fa32eb3867d764f3238005b5b81._comment
new file mode 100644
index 0000000000..fe609cab01
--- /dev/null
+++ b/doc/devblog/day_425__tor/comment_1_1dd41fa32eb3867d764f3238005b5b81._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="grawity@2ea26be48562f66fcb9b66307da72b1e2e37453f"
+ nickname="grawity"
+ avatar="http://cdn.libravatar.org/avatar/7003e967f47003bae82966aa373de8ef"
+ subject="comment 1"
+ date="2016-11-15T18:01:18Z"
+ content="""
+…or `pkexec`, which is present on many systems and generally integrates better with whatever DE/non-DE the user may be running.
+
+(OTOH, pkexec does not set up X11 access – then again, root helpers shouldn't need it.)
+"""]]

From 556b2ded2ba8270846fa207255b4c2def6ef5d8a Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 15 Nov 2016 18:26:17 -0400
Subject: [PATCH 019/367] sync: Pass --allow-unrelated-histories to git merge
 when used with git git 2.9.0 or newer.

This makes merging a remote into a freshly created direct mode repository
work the same as it works in indirect mode.

The git-annex branches would get merged in any case by a sync,
since that doesn't use git merge.

This might need to be revisited later to better mirror git's behavior.
---
 Assistant/Sync.hs           | 10 +---------
 Assistant/Threads/Merger.hs |  3 +--
 CHANGELOG                   |  3 +++
 Command/Sync.hs             |  7 ++++++-
 4 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/Assistant/Sync.hs b/Assistant/Sync.hs
index 9b9e7ebe5e..ff34c0656b 100644
--- a/Assistant/Sync.hs
+++ b/Assistant/Sync.hs
@@ -21,7 +21,6 @@ import Utility.Parallel
 import qualified Git
 import qualified Git.Command
 import qualified Git.Ref
-import qualified Git.Merge
 import qualified Remote
 import qualified Types.Remote as Remote
 import qualified Remote.List as Remote
@@ -239,19 +238,12 @@ manualPull currentbranch remotes = do
 			)
 	haddiverged <- liftAnnex Annex.Branch.forceUpdate
 	forM_ normalremotes $ \r ->
-		liftAnnex $ Command.Sync.mergeRemote r currentbranch mergeConfig
+		liftAnnex $ Command.Sync.mergeRemote r currentbranch Command.Sync.mergeConfig
 	u <- liftAnnex getUUID
 	forM_ xmppremotes $ \r ->
 		sendNetMessage $ Pushing (getXMPPClientID r) (PushRequest u)
 	return (catMaybes failed, haddiverged)
 
-mergeConfig :: [Git.Merge.MergeConfig]
-mergeConfig = 
-	[ Git.Merge.MergeNonInteractive
-	-- Pairing involves merging unrelated histories
-	, Git.Merge.MergeUnrelatedHistories
-	]
-
 {- Start syncing a remote, using a background thread. -}
 syncRemote :: Remote -> Assistant ()
 syncRemote remote = do
diff --git a/Assistant/Threads/Merger.hs b/Assistant/Threads/Merger.hs
index 521e5bda6b..ce0dfbcb42 100644
--- a/Assistant/Threads/Merger.hs
+++ b/Assistant/Threads/Merger.hs
@@ -12,7 +12,6 @@ import Assistant.TransferQueue
 import Assistant.BranchChange
 import Assistant.DaemonStatus
 import Assistant.ScanRemotes
-import Assistant.Sync
 import Utility.DirWatcher
 import Utility.DirWatcher.Types
 import qualified Annex.Branch
@@ -86,7 +85,7 @@ onChange file
 					, "into", Git.fromRef b
 					]
 				void $ liftAnnex $ Command.Sync.merge
-					currbranch mergeConfig
+					currbranch Command.Sync.mergeConfig
 					Git.Branch.AutomaticCommit
 					changedbranch
 	mergecurrent _ = noop
diff --git a/CHANGELOG b/CHANGELOG
index 33f434df46..a792d71cc7 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -6,6 +6,9 @@ git-annex (6.20161112) UNRELEASED; urgency=medium
     if so, either use stack to build, or run cabal with 
     --constraint='persistent ==2.2.4.1'
     Hopefully this mess with esqueleto will be resolved soon.
+  * sync: Pass --allow-unrelated-histories to git merge when used with git
+    git 2.9.0 or newer. This makes merging a remote into a freshly created
+    direct mode repository work the same as it works in indirect mode.
 
  -- Joey Hess <id@joeyh.name>  Tue, 15 Nov 2016 11:15:27 -0400
 
diff --git a/Command/Sync.hs b/Command/Sync.hs
index d7edac7435..fb80c3e74b 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -169,7 +169,12 @@ prepMerge :: Annex ()
 prepMerge = Annex.changeDirectory =<< fromRepo Git.repoPath
 
 mergeConfig :: [Git.Merge.MergeConfig]	
-mergeConfig = [Git.Merge.MergeNonInteractive]
+mergeConfig = 
+	[ Git.Merge.MergeNonInteractive
+	-- In several situations, unrelated histories should be merged
+	-- together. This includes pairing in the assistant etc.
+	, Git.Merge.MergeUnrelatedHistories
+	]
 
 merge :: CurrBranch -> [Git.Merge.MergeConfig] -> Git.Branch.CommitMode -> Git.Branch -> Annex Bool
 merge (Just b, Just adj) mergeconfig commitmode tomerge =

From 69915c6c9b5e41d600de052e870535845abbde96 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 15 Nov 2016 18:51:07 -0400
Subject: [PATCH 020/367] fix tricky warning with ghc 8

Whether Route was exported from Assistant.WebApp.Types
or not depended on the version of ghc. So, explictly export it.
---
 Assistant/Alert.hs        | 1 -
 Assistant/WebApp/Types.hs | 5 ++++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/Assistant/Alert.hs b/Assistant/Alert.hs
index bc79a70a8d..6db66399c5 100644
--- a/Assistant/Alert.hs
+++ b/Assistant/Alert.hs
@@ -26,7 +26,6 @@ import qualified Control.Exception as E
 import Assistant.DaemonStatus
 import Assistant.WebApp.Types
 import Assistant.WebApp (renderUrl)
-import Yesod
 #endif
 import Assistant.Monad
 import Assistant.Types.UrlRenderer
diff --git a/Assistant/WebApp/Types.hs b/Assistant/WebApp/Types.hs
index 1ab0fa306f..6b0eb9413f 100644
--- a/Assistant/WebApp/Types.hs
+++ b/Assistant/WebApp/Types.hs
@@ -10,7 +10,10 @@
 {-# LANGUAGE FlexibleInstances, FlexibleContexts, ViewPatterns #-}
 {-# OPTIONS_GHC -fno-warn-orphans #-}
 
-module Assistant.WebApp.Types where
+module Assistant.WebApp.Types (
+	module Assistant.WebApp.Types,
+	Route
+) where
 
 import Assistant.Common
 import Assistant.Ssh

From 6c1f0c6b0274248d493365a5728be41dd30d233b Mon Sep 17 00:00:00 2001
From: David_K <David_K@web>
Date: Wed, 16 Nov 2016 01:28:04 +0000
Subject: [PATCH 021/367] Added a comment

---
 ...ment_27_2b6c92994533988b7dd9e6e6abc45835._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/special_remotes/S3/comment_27_2b6c92994533988b7dd9e6e6abc45835._comment

diff --git a/doc/special_remotes/S3/comment_27_2b6c92994533988b7dd9e6e6abc45835._comment b/doc/special_remotes/S3/comment_27_2b6c92994533988b7dd9e6e6abc45835._comment
new file mode 100644
index 0000000000..4d996f8de3
--- /dev/null
+++ b/doc/special_remotes/S3/comment_27_2b6c92994533988b7dd9e6e6abc45835._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="David_K"
+ avatar="http://cdn.libravatar.org/avatar/09dd8544695feb9b8d8ee54e4ff0168d"
+ subject="comment 27"
+ date="2016-11-16T01:28:04Z"
+ content="""
+I'd like to reiterate q
+
+Is there a way to tell the S3 backend to store the files as they are named locally, instead of by hashed content name? i.e., I've annexed foo/bar.txt and annex puts it in s3 as mybucket.name/foo/bar.txt instead of mybucket.name/GPGHMACSHA1-random.txt
+
+
+"""]]

From 08ca407e676755b3e333fe198d037f76b48000d3 Mon Sep 17 00:00:00 2001
From: David_K <David_K@web>
Date: Wed, 16 Nov 2016 01:28:14 +0000
Subject: [PATCH 022/367] Added a comment

---
 ...ment_28_c4dafad82a898eafd6d9e3703fad2c48._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/special_remotes/S3/comment_28_c4dafad82a898eafd6d9e3703fad2c48._comment

diff --git a/doc/special_remotes/S3/comment_28_c4dafad82a898eafd6d9e3703fad2c48._comment b/doc/special_remotes/S3/comment_28_c4dafad82a898eafd6d9e3703fad2c48._comment
new file mode 100644
index 0000000000..864974205c
--- /dev/null
+++ b/doc/special_remotes/S3/comment_28_c4dafad82a898eafd6d9e3703fad2c48._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="David_K"
+ avatar="http://cdn.libravatar.org/avatar/09dd8544695feb9b8d8ee54e4ff0168d"
+ subject="comment 28"
+ date="2016-11-16T01:28:14Z"
+ content="""
+I'd like to reiterate a question that was unanswered above:
+
+Is there a way to tell the S3 backend to store the files as they are named locally, instead of by hashed content name? i.e., I've annexed foo/bar.txt and annex puts it in s3 as mybucket.name/foo/bar.txt instead of mybucket.name/GPGHMACSHA1-random.txt
+
+
+"""]]

From 057ee3042909a64380d5681214b029a22e63585a Mon Sep 17 00:00:00 2001
From: David_K <David_K@web>
Date: Wed, 16 Nov 2016 01:28:32 +0000
Subject: [PATCH 023/367] removed

---
 ...ment_27_2b6c92994533988b7dd9e6e6abc45835._comment | 12 ------------
 1 file changed, 12 deletions(-)
 delete mode 100644 doc/special_remotes/S3/comment_27_2b6c92994533988b7dd9e6e6abc45835._comment

diff --git a/doc/special_remotes/S3/comment_27_2b6c92994533988b7dd9e6e6abc45835._comment b/doc/special_remotes/S3/comment_27_2b6c92994533988b7dd9e6e6abc45835._comment
deleted file mode 100644
index 4d996f8de3..0000000000
--- a/doc/special_remotes/S3/comment_27_2b6c92994533988b7dd9e6e6abc45835._comment
+++ /dev/null
@@ -1,12 +0,0 @@
-[[!comment format=mdwn
- username="David_K"
- avatar="http://cdn.libravatar.org/avatar/09dd8544695feb9b8d8ee54e4ff0168d"
- subject="comment 27"
- date="2016-11-16T01:28:04Z"
- content="""
-I'd like to reiterate q
-
-Is there a way to tell the S3 backend to store the files as they are named locally, instead of by hashed content name? i.e., I've annexed foo/bar.txt and annex puts it in s3 as mybucket.name/foo/bar.txt instead of mybucket.name/GPGHMACSHA1-random.txt
-
-
-"""]]

From 0a4479b8eccc8ad9f647706af0b53dc7d192ede9 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 15 Nov 2016 21:29:54 -0400
Subject: [PATCH 024/367] Avoid backtraces on expected failures when built with
 ghc 8; only use backtraces for unexpected errors.

ghc 8 added backtraces on uncaught errors. This is great, but git-annex was
using error in many places for a error message targeted at the user, in
some known problem case. A backtrace only confuses such a message, so omit it.

Notably, commands like git annex drop that failed due to eg, numcopies,
used to use error, so had a backtrace.

This commit was sponsored by Ethan Aubin.
---
 Annex/AdjustedBranch.hs                  |  6 ++---
 Annex/Branch.hs                          |  4 +--
 Annex/Content.hs                         |  4 +--
 Annex/FileMatcher.hs                     |  2 +-
 Annex/Init.hs                            |  2 +-
 Annex/View.hs                            |  4 +--
 Assistant/Threads/Watcher.hs             |  4 +--
 Assistant/Threads/WebApp.hs              |  2 +-
 Assistant/Upgrade.hs                     |  2 +-
 Assistant/WebApp/Configurators/AWS.hs    |  8 +++---
 Assistant/WebApp/Configurators/IA.hs     |  4 +--
 Assistant/WebApp/Configurators/Local.hs  |  4 +--
 Assistant/WebApp/Configurators/Ssh.hs    |  6 ++---
 Assistant/WebApp/Configurators/WebDAV.hs |  4 +--
 Assistant/WebApp/Gpg.hs                  |  6 ++---
 CHANGELOG                                |  2 ++
 CmdLine/Action.hs                        |  2 +-
 CmdLine/Batch.hs                         |  2 +-
 CmdLine/GitAnnexShell.hs                 |  8 +++---
 CmdLine/GitAnnexShell/Checks.hs          |  6 ++---
 CmdLine/Seek.hs                          | 16 ++++++------
 Command.hs                               |  6 ++---
 Command/AddUnused.hs                     |  2 +-
 Command/AddUrl.hs                        | 10 ++++----
 Command/Assistant.hs                     |  4 +--
 Command/CheckPresentKey.hs               |  6 ++---
 Command/ContentLocation.hs               |  2 +-
 Command/Dead.hs                          |  2 +-
 Command/Describe.hs                      |  2 +-
 Command/DiffDriver.hs                    |  2 +-
 Command/Direct.hs                        |  2 +-
 Command/DropKey.hs                       |  2 +-
 Command/EnableRemote.hs                  |  4 +--
 Command/ExamineKey.hs                    |  2 +-
 Command/Expire.hs                        |  4 +--
 Command/FromKey.hs                       |  8 +++---
 Command/Fsck.hs                          |  2 +-
 Command/FuzzTest.hs                      |  2 +-
 Command/GCryptSetup.hs                   |  6 ++---
 Command/Group.hs                         |  2 +-
 Command/GroupWanted.hs                   |  2 +-
 Command/Import.hs                        |  2 +-
 Command/ImportFeed.hs                    |  4 +--
 Command/Indirect.hs                      |  4 +--
 Command/InitRemote.hs                    |  8 +++---
 Command/Lock.hs                          |  4 +--
 Command/LockContent.hs                   |  4 +--
 Command/Log.hs                           |  2 +-
 Command/MetaData.hs                      |  4 +--
 Command/Move.hs                          |  2 +-
 Command/NumCopies.hs                     |  8 +++---
 Command/PreCommit.hs                     |  2 +-
 Command/Proxy.hs                         |  2 +-
 Command/ReKey.hs                         |  4 +--
 Command/ReadPresentKey.hs                |  4 +--
 Command/RegisterUrl.hs                   |  4 +--
 Command/Reinject.hs                      |  5 ++--
 Command/ResolveMerge.hs                  |  6 ++---
 Command/Schedule.hs                      |  4 +--
 Command/SetKey.hs                        |  4 +--
 Command/SetPresentKey.hs                 |  6 ++---
 Command/Sync.hs                          |  2 +-
 Command/TestRemote.hs                    |  2 +-
 Command/TransferInfo.hs                  |  2 +-
 Command/Unannex.hs                       |  2 +-
 Command/Undo.hs                          |  2 +-
 Command/Ungroup.hs                       |  2 +-
 Command/Uninit.hs                        |  8 +++---
 Command/Unused.hs                        |  4 +--
 Command/VAdd.hs                          |  4 +--
 Command/VCycle.hs                        |  2 +-
 Command/VFilter.hs                       |  2 +-
 Command/VPop.hs                          |  2 +-
 Command/Vicfg.hs                         |  2 +-
 Command/View.hs                          |  6 ++---
 Command/Wanted.hs                        |  4 +--
 Command/WebApp.hs                        |  4 +--
 Config/Files.hs                          |  2 +-
 Creds.hs                                 |  2 +-
 Crypto.hs                                |  6 ++---
 Database/Types.hs                        |  4 +--
 Git/AutoCorrect.hs                       |  2 +-
 Git/CurrentRepo.hs                       |  2 +-
 Git/GCrypt.hs                            |  2 +-
 Limit.hs                                 |  4 +--
 Logs/Transitions.hs                      |  2 +-
 Remote.hs                                |  6 ++---
 Remote/BitTorrent.hs                     | 14 +++++------
 Remote/Bup.hs                            | 10 ++++----
 Remote/Ddar.hs                           |  4 +--
 Remote/Directory.hs                      |  8 +++---
 Remote/External.hs                       | 31 ++++++++++++------------
 Remote/GCrypt.hs                         | 14 +++++------
 Remote/Git.hs                            | 22 ++++++++---------
 Remote/Glacier.hs                        | 14 +++++------
 Remote/Helper/Chunked.hs                 |  2 +-
 Remote/Helper/Encryptable.hs             |  6 ++---
 Remote/Helper/Http.hs                    |  2 +-
 Remote/Helper/Messages.hs                |  2 +-
 Remote/Helper/Ssh.hs                     |  2 +-
 Remote/Hook.hs                           |  8 +++---
 Remote/Rsync.hs                          |  8 +++---
 Remote/S3.hs                             | 14 +++++------
 Remote/Tahoe.hs                          |  8 +++---
 Remote/Web.hs                            |  2 +-
 Remote/WebDAV.hs                         |  8 +++---
 Upgrade.hs                               |  6 ++---
 Utility/Daemon.hs                        |  4 +--
 Utility/DirWatcher/FSEvents.hs           |  2 +-
 Utility/DirWatcher/INotify.hs            |  2 +-
 Utility/Exception.hs                     | 14 ++++++++++-
 Utility/Glob.hs                          |  4 ++-
 Utility/Gpg.hs                           |  2 +-
 Utility/LockFile/PidLock.hs              |  2 +-
 Utility/Quvi.hs                          |  4 +--
 Utility/UserInfo.hs                      |  3 ++-
 116 files changed, 287 insertions(+), 270 deletions(-)

diff --git a/Annex/AdjustedBranch.hs b/Annex/AdjustedBranch.hs
index 4caf637c7e..72c07a5bc4 100644
--- a/Annex/AdjustedBranch.hs
+++ b/Annex/AdjustedBranch.hs
@@ -596,7 +596,7 @@ checkAdjustedClone = ifM isBareRepo
 				aps <- fmap commitParent <$> findAdjustingCommit (AdjBranch currbranch)
 				case aps of
 					Just [p] -> setBasisBranch basis p
-					_ -> error $ "Unable to clean up from clone of adjusted branch; perhaps you should check out " ++ Git.Ref.describe origbranch
+					_ -> giveup $ "Unable to clean up from clone of adjusted branch; perhaps you should check out " ++ Git.Ref.describe origbranch
 			ifM versionSupportsUnlockedPointers
 				( return InAdjustedClone
 				, return NeedUpgradeForAdjustedClone
@@ -610,6 +610,6 @@ isGitVersionSupported = not <$> Git.Version.older "2.2.0"
 checkVersionSupported :: Annex ()
 checkVersionSupported = do
 	unlessM versionSupportsAdjustedBranch $
-		error "Adjusted branches are only supported in v6 or newer repositories."
+		giveup "Adjusted branches are only supported in v6 or newer repositories."
 	unlessM (liftIO isGitVersionSupported) $
-		error "Your version of git is too old; upgrade it to 2.2.0 or newer to use adjusted branches."
+		giveup "Your version of git is too old; upgrade it to 2.2.0 or newer to use adjusted branches."
diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index a426c76d85..9663311d51 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -225,7 +225,7 @@ getHistorical date file =
 	-- This check avoids some ugly error messages when the reflog
 	-- is empty.
 	ifM (null <$> inRepo (Git.RefLog.get' [Param (fromRef fullname), Param "-n1"]))
-		( error ("No reflog for " ++ fromRef fullname)
+		( giveup ("No reflog for " ++ fromRef fullname)
 		, getRef (Git.Ref.dateRef fullname date) file
 		)
 
@@ -574,7 +574,7 @@ checkBranchDifferences ref = do
 		<$> catFile ref differenceLog
 	mydiffs <- annexDifferences <$> Annex.getGitConfig
 	when (theirdiffs /= mydiffs) $
-		error "Remote repository is tuned in incompatable way; cannot be merged with local repository."
+		giveup "Remote repository is tuned in incompatable way; cannot be merged with local repository."
 
 ignoreRefs :: [Git.Sha] -> Annex ()
 ignoreRefs rs = do
diff --git a/Annex/Content.hs b/Annex/Content.hs
index cb96a0068b..e879e4eebb 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -268,8 +268,8 @@ lockContentUsing locker key a = do
 		(unlock lockfile)
 		(const a)
   where
-	alreadylocked = error "content is locked"
-	failedtolock e = error $ "failed to lock content: " ++ show e
+	alreadylocked = giveup "content is locked"
+	failedtolock e = giveup $ "failed to lock content: " ++ show e
 
 	lock contentfile lockfile =
 		(maybe alreadylocked return 
diff --git a/Annex/FileMatcher.hs b/Annex/FileMatcher.hs
index fa46e64b1d..654c5a9606 100644
--- a/Annex/FileMatcher.hs
+++ b/Annex/FileMatcher.hs
@@ -165,7 +165,7 @@ largeFilesMatcher = go =<< annexLargeFiles <$> Annex.getGitConfig
 	mkmatcher expr = do
 		parser <- mkLargeFilesParser
 		either badexpr return $ parsedToMatcher $ parser expr
-	badexpr e = error $ "bad annex.largefiles configuration: " ++ e
+	badexpr e = giveup $ "bad annex.largefiles configuration: " ++ e
 
 simply :: MatchFiles Annex -> ParseResult
 simply = Right . Operation
diff --git a/Annex/Init.hs b/Annex/Init.hs
index 5aff4cf398..8a208fe2bb 100644
--- a/Annex/Init.hs
+++ b/Annex/Init.hs
@@ -129,7 +129,7 @@ ensureInitialized = getVersion >>= maybe needsinit checkUpgrade
   where
 	needsinit = ifM Annex.Branch.hasSibling
 			( initialize Nothing Nothing
-			, error "First run: git-annex init"
+			, giveup "First run: git-annex init"
 			)
 
 {- Checks if a repository is initialized. Does not check version for ugrade. -}
diff --git a/Annex/View.hs b/Annex/View.hs
index 7d2b43e608..d865c8f783 100644
--- a/Annex/View.hs
+++ b/Annex/View.hs
@@ -110,7 +110,7 @@ refineView origview = checksize . calc Unchanged origview
 			in (view', Narrowing)
 	
 	checksize r@(v, _)
-		| viewTooLarge v = error $ "View is too large (" ++ show (visibleViewSize v) ++ " levels of subdirectories)"
+		| viewTooLarge v = giveup $ "View is too large (" ++ show (visibleViewSize v) ++ " levels of subdirectories)"
 		| otherwise = r
 
 updateViewComponent :: ViewComponent -> MetaField -> ViewFilter -> Writer [ViewChange] ViewComponent
@@ -424,4 +424,4 @@ genViewBranch view = withViewIndex $ do
 	return branch
 
 withCurrentView :: (View -> Annex a) -> Annex a
-withCurrentView a = maybe (error "Not in a view.") a =<< currentView
+withCurrentView a = maybe (giveup "Not in a view.") a =<< currentView
diff --git a/Assistant/Threads/Watcher.hs b/Assistant/Threads/Watcher.hs
index 1f50065b9c..4b82a799d7 100644
--- a/Assistant/Threads/Watcher.hs
+++ b/Assistant/Threads/Watcher.hs
@@ -65,10 +65,10 @@ checkCanWatch
 #else
 		noop
 #endif
-	| otherwise = error "watch mode is not available on this system"
+	| otherwise = giveup "watch mode is not available on this system"
 
 needLsof :: Annex ()
-needLsof = error $ unlines
+needLsof = giveup $ unlines
 	[ "The lsof command is needed for watch mode to be safe, and is not in PATH."
 	, "To override lsof checks to ensure that files are not open for writing"
 	, "when added to the annex, you can use --force"
diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 58effdc1c0..f9a456f357 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -71,7 +71,7 @@ webAppThread assistantdata urlrenderer noannex cannotrun postfirstrun listenhost
 #ifdef __ANDROID__
 	when (isJust listenhost') $
 		-- See Utility.WebApp
-		error "Sorry, --listen is not currently supported on Android"
+		giveup "Sorry, --listen is not currently supported on Android"
 #endif
 	webapp <- WebApp
 		<$> pure assistantdata
diff --git a/Assistant/Upgrade.hs b/Assistant/Upgrade.hs
index fa5870f3ea..28b838dc2b 100644
--- a/Assistant/Upgrade.hs
+++ b/Assistant/Upgrade.hs
@@ -153,7 +153,7 @@ upgradeToDistribution newdir cleanup distributionfile = do
   where
 	changeprogram program = liftIO $ do
 		unlessM (boolSystem program [Param "version"]) $
-			error "New git-annex program failed to run! Not using."
+			giveup "New git-annex program failed to run! Not using."
 		pf <- programFile
 		liftIO $ writeFile pf program
 	
diff --git a/Assistant/WebApp/Configurators/AWS.hs b/Assistant/WebApp/Configurators/AWS.hs
index bec3123f89..981477e114 100644
--- a/Assistant/WebApp/Configurators/AWS.hs
+++ b/Assistant/WebApp/Configurators/AWS.hs
@@ -139,7 +139,7 @@ postAddS3R = awsConfigurator $ do
 				]
 		_ -> $(widgetFile "configurators/adds3")
 #else
-postAddS3R = error "S3 not supported by this build"
+postAddS3R = giveup "S3 not supported by this build"
 #endif
 
 getAddGlacierR :: Handler Html
@@ -161,7 +161,7 @@ postAddGlacierR = glacierConfigurator $ do
 				]
 		_ -> $(widgetFile "configurators/addglacier")
 #else
-postAddGlacierR = error "S3 not supported by this build"
+postAddGlacierR = giveup "S3 not supported by this build"
 #endif
 
 getEnableS3R :: UUID -> Handler Html
@@ -179,7 +179,7 @@ postEnableS3R :: UUID -> Handler Html
 #ifdef WITH_S3
 postEnableS3R uuid = awsConfigurator $ enableAWSRemote S3.remote uuid
 #else
-postEnableS3R _ = error "S3 not supported by this build"
+postEnableS3R _ = giveup "S3 not supported by this build"
 #endif
 
 getEnableGlacierR :: UUID -> Handler Html
@@ -205,7 +205,7 @@ enableAWSRemote remotetype uuid = do
 				T.pack <$> Remote.prettyUUID uuid
 			$(widgetFile "configurators/enableaws")
 #else
-enableAWSRemote _ _ = error "S3 not supported by this build"
+enableAWSRemote _ _ = giveup "S3 not supported by this build"
 #endif
 
 makeAWSRemote :: SpecialRemoteMaker -> RemoteType -> StandardGroup -> AWSCreds -> RemoteName -> RemoteConfig -> Handler ()
diff --git a/Assistant/WebApp/Configurators/IA.hs b/Assistant/WebApp/Configurators/IA.hs
index a6816958e7..c46fcf510e 100644
--- a/Assistant/WebApp/Configurators/IA.hs
+++ b/Assistant/WebApp/Configurators/IA.hs
@@ -147,7 +147,7 @@ postAddIAR = iaConfigurator $ do
 					]
 		_ -> $(widgetFile "configurators/addia")
 #else
-postAddIAR = error "S3 not supported by this build"
+postAddIAR = giveup "S3 not supported by this build"
 #endif
 
 getEnableIAR :: UUID -> Handler Html
@@ -157,7 +157,7 @@ postEnableIAR :: UUID -> Handler Html
 #ifdef WITH_S3
 postEnableIAR = iaConfigurator . enableIARemote
 #else
-postEnableIAR _ = error "S3 not supported by this build"
+postEnableIAR _ = giveup "S3 not supported by this build"
 #endif
 
 #ifdef WITH_S3
diff --git a/Assistant/WebApp/Configurators/Local.hs b/Assistant/WebApp/Configurators/Local.hs
index 76f21e9937..f2079c0ed2 100644
--- a/Assistant/WebApp/Configurators/Local.hs
+++ b/Assistant/WebApp/Configurators/Local.hs
@@ -151,7 +151,7 @@ getFirstRepositoryR = postFirstRepositoryR
 postFirstRepositoryR :: Handler Html
 postFirstRepositoryR = page "Getting started" (Just Configuration) $ do
 	unlessM (liftIO $ inPath "git") $
-		error "You need to install git in order to use git-annex!"
+		giveup "You need to install git in order to use git-annex!"
 #ifdef __ANDROID__
 	androidspecial <- liftIO $ doesDirectoryExist "/sdcard/DCIM"
 	let path = "/sdcard/annex"
@@ -309,7 +309,7 @@ getFinishAddDriveR drive = go
 		mu <- liftAnnex $ probeGCryptRemoteUUID dir
 		case mu of
 			Just u -> enableexistinggcryptremote u
-			Nothing -> error "The drive contains a gcrypt repository that is not a git-annex special remote. This is not supported."
+			Nothing -> giveup "The drive contains a gcrypt repository that is not a git-annex special remote. This is not supported."
 	enableexistinggcryptremote u = do
 		remotename' <- liftAnnex $ getGCryptRemoteName u dir
 		makewith $ const $ do
diff --git a/Assistant/WebApp/Configurators/Ssh.hs b/Assistant/WebApp/Configurators/Ssh.hs
index 5ad28402e4..950290249c 100644
--- a/Assistant/WebApp/Configurators/Ssh.hs
+++ b/Assistant/WebApp/Configurators/Ssh.hs
@@ -196,7 +196,7 @@ postEnableSshGCryptR u = whenGcryptInstalled $
 	enablegcrypt sshdata _ = prepSsh False sshdata $ \sshdata' ->
 		sshConfigurator $
 			checkExistingGCrypt sshdata' $
-				error "Expected to find an encrypted git repository, but did not."
+				giveup "Expected to find an encrypted git repository, but did not."
 	getsshinput = parseSshUrl <=< M.lookup "gitrepo"
 
 getEnableSshGitRemoteR :: UUID -> Handler Html
@@ -475,7 +475,7 @@ checkExistingGCrypt sshdata nope = checkGCryptRepoEncryption repourl nope nope $
 	case mu of
 		Just u -> void $ liftH $
 			combineExistingGCrypt sshdata u
-		Nothing -> error "The location contains a gcrypt repository that is not a git-annex special remote. This is not supported."
+		Nothing -> giveup "The location contains a gcrypt repository that is not a git-annex special remote. This is not supported."
   where
 	repourl = genSshUrl sshdata
 
@@ -641,7 +641,7 @@ enableRsyncNetGCrypt sshinput reponame =
 		checkGCryptRepoEncryption (genSshUrl sshdata) notencrypted notinstalled $
 			enableGCrypt sshdata reponame
   where
-	notencrypted = error "Unexpectedly found a non-encrypted git repository, instead of the expected encrypted git repository."
+	notencrypted = giveup "Unexpectedly found a non-encrypted git repository, instead of the expected encrypted git repository."
 	notinstalled = error "internal"
 
 {- Prepares rsync.net ssh key and creates the directory that will be 
diff --git a/Assistant/WebApp/Configurators/WebDAV.hs b/Assistant/WebApp/Configurators/WebDAV.hs
index 613e5439a7..9c168d744f 100644
--- a/Assistant/WebApp/Configurators/WebDAV.hs
+++ b/Assistant/WebApp/Configurators/WebDAV.hs
@@ -82,7 +82,7 @@ postAddBoxComR = boxConfigurator $ do
 				]
 		_ -> $(widgetFile "configurators/addbox.com")
 #else
-postAddBoxComR = error "WebDAV not supported by this build"
+postAddBoxComR = giveup "WebDAV not supported by this build"
 #endif
 
 getEnableWebDAVR :: UUID -> Handler Html
@@ -120,7 +120,7 @@ postEnableWebDAVR uuid = do
 					T.pack <$> Remote.prettyUUID uuid
 				$(widgetFile "configurators/enablewebdav")
 #else
-postEnableWebDAVR _ = error "WebDAV not supported by this build"
+postEnableWebDAVR _ = giveup "WebDAV not supported by this build"
 #endif
 
 #ifdef WITH_WEBDAV
diff --git a/Assistant/WebApp/Gpg.hs b/Assistant/WebApp/Gpg.hs
index 6afb20fd11..10223ccccb 100644
--- a/Assistant/WebApp/Gpg.hs
+++ b/Assistant/WebApp/Gpg.hs
@@ -56,7 +56,7 @@ withNewSecretKey use = do
 	liftIO $ genSecretKey cmd RSA "" userid maxRecommendedKeySize
 	results <- M.keys . M.filter (== userid) <$> liftIO (secretKeys cmd)
 	case results of
-		[] -> error "Failed to generate gpg key!"
+		[] -> giveup "Failed to generate gpg key!"
 		(key:_) -> use key
 
 {- Tries to find the name used in remote.log for a gcrypt repository
@@ -85,7 +85,7 @@ getGCryptRemoteName u repoloc = do
 	void $ inRepo $ Git.Remote.Remove.remove tmpremote
 	maybe missing return mname
   where
-	missing = error $ "Cannot find configuration for the gcrypt remote at " ++ repoloc
+	missing = giveup $ "Cannot find configuration for the gcrypt remote at " ++ repoloc
 
 {- Checks to see if a repo is encrypted with gcrypt, and runs one action if
  - it's not an another if it is.
@@ -103,7 +103,7 @@ checkGCryptRepoEncryption location notencrypted notinstalled encrypted =
 	dispatch Git.GCrypt.Decryptable = encrypted
 	dispatch Git.GCrypt.NotEncrypted = notencrypted
 	dispatch Git.GCrypt.NotDecryptable =
-		error "This git repository is encrypted with a GnuPG key that you do not have."
+		giveup "This git repository is encrypted with a GnuPG key that you do not have."
 
 {- Gets the UUID of the gcrypt repo at a location, which may not exist.
  - Only works if the gcrypt repo was created as a git-annex remote. -}
diff --git a/CHANGELOG b/CHANGELOG
index a792d71cc7..71ef1c1002 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -9,6 +9,8 @@ git-annex (6.20161112) UNRELEASED; urgency=medium
   * sync: Pass --allow-unrelated-histories to git merge when used with git
     git 2.9.0 or newer. This makes merging a remote into a freshly created
     direct mode repository work the same as it works in indirect mode.
+  * Avoid backtraces on expected failures when built with ghc 8;
+    only use backtraces for unexpected errors.
 
  -- Joey Hess <id@joeyh.name>  Tue, 15 Nov 2016 11:15:27 -0400
 
diff --git a/CmdLine/Action.hs b/CmdLine/Action.hs
index 7d9dce5743..27621e4458 100644
--- a/CmdLine/Action.hs
+++ b/CmdLine/Action.hs
@@ -38,7 +38,7 @@ performCommandAction Command { cmdcheck = c, cmdname = name } seek cont = do
 	showerrcount =<< Annex.getState Annex.errcounter
   where
 	showerrcount 0 = noop
-	showerrcount cnt = error $ name ++ ": " ++ show cnt ++ " failed"
+	showerrcount cnt = giveup $ name ++ ": " ++ show cnt ++ " failed"
 
 {- Runs one of the actions needed to perform a command.
  - Individual actions can fail without stopping the whole command,
diff --git a/CmdLine/Batch.hs b/CmdLine/Batch.hs
index cca93b0b39..627c1df101 100644
--- a/CmdLine/Batch.hs
+++ b/CmdLine/Batch.hs
@@ -56,7 +56,7 @@ batchInput parser a = do
 			either parseerr a (parser v)
 			batchInput parser a
   where
-	parseerr s = error $ "Batch input parse failure: " ++ s
+	parseerr s = giveup $ "Batch input parse failure: " ++ s
 
 -- Runs a CommandStart in batch mode.
 --
diff --git a/CmdLine/GitAnnexShell.hs b/CmdLine/GitAnnexShell.hs
index 599d12fec1..70c86ec2f7 100644
--- a/CmdLine/GitAnnexShell.hs
+++ b/CmdLine/GitAnnexShell.hs
@@ -71,7 +71,7 @@ globalOptions =
 		check Nothing = unexpected expected "uninitialized repository"
 		check (Just u) = unexpectedUUID expected u
 	unexpectedUUID expected u = unexpected expected $ "UUID " ++ fromUUID u
-	unexpected expected s = error $
+	unexpected expected s = giveup $
 		"expected repository UUID " ++ expected ++ " but found " ++ s
 
 run :: [String] -> IO ()
@@ -109,7 +109,7 @@ builtin cmd dir params = do
 		Git.Config.read r
 			`catchIO` \_ -> do
 				hn <- fromMaybe "unknown" <$> getHostname
-				error $ "failed to read git config of git repository in " ++ hn ++ " on " ++ dir ++ "; perhaps this repository is not set up correctly or has moved"
+				giveup $ "failed to read git config of git repository in " ++ hn ++ " on " ++ dir ++ "; perhaps this repository is not set up correctly or has moved"
 
 external :: [String] -> IO ()
 external params = do
@@ -120,7 +120,7 @@ external params = do
 	checkDirectory lastparam
 	checkNotLimited
 	unlessM (boolSystem "git-shell" $ map Param $ "-c":params') $
-		error "git-shell failed"
+		giveup "git-shell failed"
 
 {- Split the input list into 3 groups separated with a double dash --.
  - Parameters between two -- markers are field settings, in the form:
@@ -150,6 +150,6 @@ checkField (field, val)
 	| otherwise = False
 
 failure :: IO ()
-failure = error $ "bad parameters\n\n" ++ usage h cmds
+failure = giveup $ "bad parameters\n\n" ++ usage h cmds
   where
 	h = "git-annex-shell [-c] command [parameters ...] [option ...]"
diff --git a/CmdLine/GitAnnexShell/Checks.hs b/CmdLine/GitAnnexShell/Checks.hs
index 63d2e594f7..47bc11a767 100644
--- a/CmdLine/GitAnnexShell/Checks.hs
+++ b/CmdLine/GitAnnexShell/Checks.hs
@@ -26,7 +26,7 @@ checkEnv var = do
 	case v of
 		Nothing -> noop
 		Just "" -> noop
-		Just _ -> error $ "Action blocked by " ++ var
+		Just _ -> giveup $ "Action blocked by " ++ var
 
 checkDirectory :: Maybe FilePath -> IO ()
 checkDirectory mdir = do
@@ -44,7 +44,7 @@ checkDirectory mdir = do
 					then noop
 					else req d' (Just dir')
   where
-	req d mdir' = error $ unwords 
+	req d mdir' = giveup $ unwords 
 		[ "Only allowed to access"
 		, d
 		, maybe "and could not determine directory from command line" ("not " ++) mdir'
@@ -64,4 +64,4 @@ gitAnnexShellCheck :: Command -> Command
 gitAnnexShellCheck = addCheck okforshell . dontCheck repoExists
   where
 	okforshell = unlessM (isInitialized <||> isJust . gcryptId <$> Annex.getGitConfig) $
-		error "Not a git-annex or gcrypt repository."
+		giveup "Not a git-annex or gcrypt repository."
diff --git a/CmdLine/Seek.hs b/CmdLine/Seek.hs
index 5d20ad0dbe..7fc64c5284 100644
--- a/CmdLine/Seek.hs
+++ b/CmdLine/Seek.hs
@@ -40,7 +40,7 @@ withFilesInGitNonRecursive :: String -> (FilePath -> CommandStart) -> CmdParams
 withFilesInGitNonRecursive needforce a params = ifM (Annex.getState Annex.force)
 	( withFilesInGit a params
 	, if null params
-		then error needforce
+		then giveup needforce
 		else seekActions $ prepFiltered a (getfiles [] params)
 	)
   where
@@ -54,7 +54,7 @@ withFilesInGitNonRecursive needforce a params = ifM (Annex.getState Annex.force)
 			[] -> do
 				void $ liftIO $ cleanup
 				getfiles c ps
-			_ -> error needforce
+			_ -> giveup needforce
 
 withFilesNotInGit :: Bool -> (FilePath -> CommandStart) -> CmdParams -> CommandSeek
 withFilesNotInGit skipdotfiles a params
@@ -117,7 +117,7 @@ withPairs a params = seekActions $ return $ map a $ pairs [] params
   where
 	pairs c [] = reverse c
 	pairs c (x:y:xs) = pairs ((x,y):c) xs
-	pairs _ _ = error "expected pairs"
+	pairs _ _ = giveup "expected pairs"
 
 withFilesToBeCommitted :: (FilePath -> CommandStart) -> CmdParams -> CommandSeek
 withFilesToBeCommitted a params = seekActions $ prepFiltered a $
@@ -152,11 +152,11 @@ withFilesMaybeModified a params = seekActions $
 withKeys :: (Key -> CommandStart) -> CmdParams -> CommandSeek
 withKeys a params = seekActions $ return $ map (a . parse) params
   where
-	parse p = fromMaybe (error "bad key") $ file2key p
+	parse p = fromMaybe (giveup "bad key") $ file2key p
 
 withNothing :: CommandStart -> CmdParams -> CommandSeek
 withNothing a [] = seekActions $ return [a]
-withNothing _ _ = error "This command takes no parameters."
+withNothing _ _ = giveup "This command takes no parameters."
 
 {- Handles the --all, --branch, --unused, --failed, --key, and
  - --incomplete options, which specify particular keys to run an
@@ -191,7 +191,7 @@ withKeyOptions'
 withKeyOptions' ko auto mkkeyaction fallbackaction params = do
 	bare <- fromRepo Git.repoIsLocalBare
 	when (auto && bare) $
-		error "Cannot use --auto in a bare repository"
+		giveup "Cannot use --auto in a bare repository"
 	case (null params, ko) of
 		(True, Nothing)
 			| bare -> noauto $ runkeyaction loggedKeys
@@ -203,10 +203,10 @@ withKeyOptions' ko auto mkkeyaction fallbackaction params = do
 		(True, Just (WantSpecificKey k)) -> noauto $ runkeyaction (return [k])
 		(True, Just WantIncompleteKeys) -> noauto $ runkeyaction incompletekeys
 		(True, Just (WantBranchKeys bs)) -> noauto $ runbranchkeys bs
-		(False, Just _) -> error "Can only specify one of file names, --all, --branch, --unused, --failed, --key, or --incomplete"
+		(False, Just _) -> giveup "Can only specify one of file names, --all, --branch, --unused, --failed, --key, or --incomplete"
   where
 	noauto a
-		| auto = error "Cannot use --auto with --all or --branch or --unused or --key or --incomplete"
+		| auto = giveup "Cannot use --auto with --all or --branch or --unused or --key or --incomplete"
 		| otherwise = a
 	incompletekeys = staleKeysPrune gitAnnexTmpObjectDir True
 	runkeyaction getks = do
diff --git a/Command.hs b/Command.hs
index 94a4742577..f8d4fe32bb 100644
--- a/Command.hs
+++ b/Command.hs
@@ -101,15 +101,15 @@ repoExists = CommandCheck 0 ensureInitialized
 
 notDirect :: Command -> Command
 notDirect = addCheck $ whenM isDirect $
-	error "You cannot run this command in a direct mode repository."
+	giveup "You cannot run this command in a direct mode repository."
 
 notBareRepo :: Command -> Command
 notBareRepo = addCheck $ whenM (fromRepo Git.repoIsLocalBare) $
-	error "You cannot run this command in a bare repository."
+	giveup "You cannot run this command in a bare repository."
 
 noDaemonRunning :: Command -> Command
 noDaemonRunning = addCheck $ whenM (isJust <$> daemonpid) $
-	error "You cannot run this command while git-annex watch or git-annex assistant is running."
+	giveup "You cannot run this command while git-annex watch or git-annex assistant is running."
   where
 	daemonpid = liftIO . checkDaemon =<< fromRepo gitAnnexPidFile
 
diff --git a/Command/AddUnused.hs b/Command/AddUnused.hs
index 7a9a1ba30c..c83c74e726 100644
--- a/Command/AddUnused.hs
+++ b/Command/AddUnused.hs
@@ -38,4 +38,4 @@ perform key = next $ do
  - it seems better to error out, rather than moving bad/tmp content into
  - the annex. -}
 performOther :: String -> Key -> CommandPerform
-performOther other _ = error $ "cannot addunused " ++ other ++ "content"
+performOther other _ = giveup $ "cannot addunused " ++ other ++ "content"
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 80f3582ed5..e32ceb5684 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -133,7 +133,7 @@ checkUrl r o u = do
 				let f' = adjustFile o (deffile </> fromSafeFilePath f)
 				void $ commandAction $
 					startRemote r (relaxedOption o) f' u' sz
-		| otherwise = error $ unwords
+		| otherwise = giveup $ unwords
 			[ "That url contains multiple files according to the"
 			, Remote.name r
 			, " remote; cannot add it to a single file."
@@ -182,7 +182,7 @@ startWeb :: AddUrlOptions -> String -> CommandStart
 startWeb o s = go $ fromMaybe bad $ parseURI urlstring
   where
 	(urlstring, downloader) = getDownloader s
-	bad = fromMaybe (error $ "bad url " ++ urlstring) $
+	bad = fromMaybe (giveup $ "bad url " ++ urlstring) $
 		Url.parseURIRelaxed $ urlstring
 	go url = case downloader of
 		QuviDownloader -> usequvi
@@ -208,7 +208,7 @@ startWeb o s = go $ fromMaybe bad $ parseURI urlstring
 						)
 		showStart "addurl" file
 		next $ performWeb (relaxedOption o) urlstring file urlinfo
-	badquvi = error $ "quvi does not know how to download url " ++ urlstring
+	badquvi = giveup $ "quvi does not know how to download url " ++ urlstring
 	usequvi = do
 		page <- fromMaybe badquvi
 			<$> withQuviOptions Quvi.forceQuery [Quvi.quiet, Quvi.httponly] urlstring
@@ -372,7 +372,7 @@ url2file url pathdepth pathmax = case pathdepth of
 		| depth >= length urlbits -> frombits id
 		| depth > 0 -> frombits $ drop depth
 		| depth < 0 -> frombits $ reverse . take (negate depth) . reverse
-		| otherwise -> error "bad --pathdepth"
+		| otherwise -> giveup "bad --pathdepth"
   where
 	fullurl = concat
 		[ maybe "" uriRegName (uriAuthority url)
@@ -385,7 +385,7 @@ url2file url pathdepth pathmax = case pathdepth of
 
 urlString2file :: URLString -> Maybe Int -> Int -> FilePath
 urlString2file s pathdepth pathmax = case Url.parseURIRelaxed s of
-	Nothing -> error $ "bad uri " ++ s
+	Nothing -> giveup $ "bad uri " ++ s
 	Just u -> url2file u pathdepth pathmax
 
 adjustFile :: AddUrlOptions -> FilePath -> FilePath
diff --git a/Command/Assistant.hs b/Command/Assistant.hs
index 690f36f199..6a9ae6436a 100644
--- a/Command/Assistant.hs
+++ b/Command/Assistant.hs
@@ -66,14 +66,14 @@ startNoRepo :: AssistantOptions -> IO ()
 startNoRepo o
 	| autoStartOption o = autoStart o
 	| autoStopOption o = autoStop
-	| otherwise = error "Not in a git repository."
+	| otherwise = giveup "Not in a git repository."
 
 autoStart :: AssistantOptions -> IO ()
 autoStart o = do
 	dirs <- liftIO readAutoStartFile
 	when (null dirs) $ do
 		f <- autoStartFile
-		error $ "Nothing listed in " ++ f
+		giveup $ "Nothing listed in " ++ f
 	program <- programPath
 	haveionice <- pure Build.SysConfig.ionice <&&> inPath "ionice"
 	forM_ dirs $ \d -> do
diff --git a/Command/CheckPresentKey.hs b/Command/CheckPresentKey.hs
index 29df810a63..4f9b4b1207 100644
--- a/Command/CheckPresentKey.hs
+++ b/Command/CheckPresentKey.hs
@@ -40,7 +40,7 @@ seek o = case batchOption o of
 			_ -> wrongnumparams
 		batchInput Right $ checker >=> batchResult
   where
-	wrongnumparams = error "Wrong number of parameters"
+	wrongnumparams = giveup "Wrong number of parameters"
 					
 data Result = Present | NotPresent | CheckFailure String
 
@@ -71,8 +71,8 @@ batchResult Present = liftIO $ putStrLn "1"
 batchResult _ = liftIO $ putStrLn "0"
 
 toKey :: String -> Key
-toKey = fromMaybe (error "Bad key") . file2key
+toKey = fromMaybe (giveup "Bad key") . file2key
 
 toRemote :: String -> Annex Remote
-toRemote rn = maybe (error "Unknown remote") return
+toRemote rn = maybe (giveup "Unknown remote") return
 	=<< Remote.byNameWithUUID (Just rn)
diff --git a/Command/ContentLocation.hs b/Command/ContentLocation.hs
index 5b2acb6a54..202d76a21d 100644
--- a/Command/ContentLocation.hs
+++ b/Command/ContentLocation.hs
@@ -19,7 +19,7 @@ cmd = noCommit $ noMessages $
 
 run :: () -> String -> Annex Bool
 run _ p = do
-	let k = fromMaybe (error "bad key") $ file2key p
+	let k = fromMaybe (giveup "bad key") $ file2key p
 	maybe (return False) (\f -> liftIO (putStrLn f) >> return True)
 		=<< inAnnex' (pure True) Nothing check k
   where
diff --git a/Command/Dead.hs b/Command/Dead.hs
index ecbe412938..44cf7b7f6a 100644
--- a/Command/Dead.hs
+++ b/Command/Dead.hs
@@ -37,7 +37,7 @@ startKey key = do
 	ls <- keyLocations key
 	case ls of
 		[] -> next $ performKey key
-		_ -> error "This key is still known to be present in some locations; not marking as dead."
+		_ -> giveup "This key is still known to be present in some locations; not marking as dead."
 		
 performKey :: Key -> CommandPerform
 performKey key = do
diff --git a/Command/Describe.hs b/Command/Describe.hs
index 8872244f0f..dc7a5d8f91 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -25,7 +25,7 @@ start (name:description) = do
 	showStart "describe" name
 	u <- Remote.nameToUUID name
 	next $ perform u $ unwords description
-start _ = error "Specify a repository and a description."	
+start _ = giveup "Specify a repository and a description."	
 
 perform :: UUID -> String -> CommandPerform
 perform u description = do
diff --git a/Command/DiffDriver.hs b/Command/DiffDriver.hs
index 2c9b4a39dc..1164dd103b 100644
--- a/Command/DiffDriver.hs
+++ b/Command/DiffDriver.hs
@@ -73,7 +73,7 @@ parseReq opts = case separate (== "--") opts of
 	mk (unmergedpath:[]) = UnmergedReq { rPath = unmergedpath }
 	mk _ = badopts
 
-	badopts = error $ "Unexpected input: " ++ unwords opts
+	badopts = giveup $ "Unexpected input: " ++ unwords opts
 
 {- Check if either file is a symlink to a git-annex object,
  - which git-diff will leave as a normal file containing the link text.
diff --git a/Command/Direct.hs b/Command/Direct.hs
index 32d63f0598..06adf0e05b 100644
--- a/Command/Direct.hs
+++ b/Command/Direct.hs
@@ -26,7 +26,7 @@ seek = withNothing start
 start :: CommandStart
 start = ifM versionSupportsDirectMode
 	( ifM isDirect ( stop , next perform )
-	, error "Direct mode is not suppported by this repository version. Use git-annex unlock instead."
+	, giveup "Direct mode is not suppported by this repository version. Use git-annex unlock instead."
 	)
 
 perform :: CommandPerform
diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index 42516f838c..65446ba06e 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -32,7 +32,7 @@ optParser desc = DropKeyOptions
 seek :: DropKeyOptions -> CommandSeek
 seek o = do
 	unlessM (Annex.getState Annex.force) $
-		error "dropkey can cause data loss; use --force if you're sure you want to do this"
+		giveup "dropkey can cause data loss; use --force if you're sure you want to do this"
 	withKeys start (toDrop o)
 	case batchOption o of
 		Batch -> batchInput parsekey $ batchCommandAction . start
diff --git a/Command/EnableRemote.hs b/Command/EnableRemote.hs
index dc3e7bc56e..e1af8bb7a4 100644
--- a/Command/EnableRemote.hs
+++ b/Command/EnableRemote.hs
@@ -63,7 +63,7 @@ startSpecialRemote name config Nothing = do
 		_ -> unknownNameError "Unknown remote name."
 startSpecialRemote name config (Just (u, c)) = do
 	let fullconfig = config `M.union` c	
-	t <- either error return (Annex.SpecialRemote.findType fullconfig)
+	t <- either giveup return (Annex.SpecialRemote.findType fullconfig)
 	showStart "enableremote" name
 	gc <- maybe def Remote.gitconfig <$> Remote.byUUID u
 	next $ performSpecialRemote t u fullconfig gc
@@ -94,7 +94,7 @@ unknownNameError prefix = do
 	disabledremotes <- filterM isdisabled =<< Annex.fromRepo Git.remotes
 	let remotesmsg = unlines $ map ("\t" ++) $
 		mapMaybe Git.remoteName disabledremotes
-	error $ concat $ filter (not . null) [prefix ++ "\n", remotesmsg, specialmsg]
+	giveup $ concat $ filter (not . null) [prefix ++ "\n", remotesmsg, specialmsg]
   where
 	isdisabled r = anyM id
 		[ (==) NoUUID <$> getRepoUUID r
diff --git a/Command/ExamineKey.hs b/Command/ExamineKey.hs
index e14ac10b8e..24d6942fe5 100644
--- a/Command/ExamineKey.hs
+++ b/Command/ExamineKey.hs
@@ -21,6 +21,6 @@ cmd = noCommit $ noMessages $ dontCheck repoExists $
 
 run :: Maybe Utility.Format.Format -> String -> Annex Bool
 run format p = do
-	let k = fromMaybe (error "bad key") $ file2key p
+	let k = fromMaybe (giveup "bad key") $ file2key p
 	showFormatted format (key2file k) (keyVars k)
 	return True
diff --git a/Command/Expire.hs b/Command/Expire.hs
index fafee4506e..8dd0e962e5 100644
--- a/Command/Expire.hs
+++ b/Command/Expire.hs
@@ -92,7 +92,7 @@ start (Expire expire) noact actlog descs u =
 data Expire = Expire (M.Map (Maybe UUID) (Maybe POSIXTime))
 
 parseExpire :: [String] -> Annex Expire
-parseExpire [] = error "Specify an expire time."
+parseExpire [] = giveup "Specify an expire time."
 parseExpire ps = do
 	now <- liftIO getPOSIXTime
 	Expire . M.fromList <$> mapM (parse now) ps
@@ -104,7 +104,7 @@ parseExpire ps = do
 			return (Just r, parsetime now t)
 	parsetime _ "never" = Nothing
 	parsetime now s = case parseDuration s of
-		Nothing -> error $ "bad expire time: " ++ s
+		Nothing -> giveup $ "bad expire time: " ++ s
 		Just d -> Just (now - durationToPOSIXTime d)
 
 parseActivity :: Monad m => String -> m Activity
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index 36cc1d31fd..670e9e6a6b 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -33,14 +33,14 @@ start force (keyname:file:[]) = do
 	let key = mkKey keyname
 	unless force $ do
 		inbackend <- inAnnex key
-		unless inbackend $ error $
+		unless inbackend $ giveup $
 			"key ("++ keyname ++") is not present in backend (use --force to override this sanity check)"
 	showStart "fromkey" file
 	next $ perform key file
 start _ [] = do
 	showStart "fromkey" "stdin"
 	next massAdd
-start _ _ = error "specify a key and a dest file"
+start _ _ = giveup "specify a key and a dest file"
 
 massAdd :: CommandPerform
 massAdd = go True =<< map (separate (== ' ')) . lines <$> liftIO getContents
@@ -51,7 +51,7 @@ massAdd = go True =<< map (separate (== ' ')) . lines <$> liftIO getContents
 		ok <- perform' key f
 		let !status' = status && ok
 		go status' rest
-	go _ _ = error "Expected pairs of key and file on stdin, but got something else."
+	go _ _ = giveup "Expected pairs of key and file on stdin, but got something else."
 
 -- From user input to a Key.
 -- User can input either a serialized key, or an url.
@@ -66,7 +66,7 @@ mkKey s = case parseURI s of
 		Backend.URL.fromUrl s Nothing
 	_ -> case file2key s of
 		Just k -> k
-		Nothing -> error $ "bad key/url " ++ s
+		Nothing -> giveup $ "bad key/url " ++ s
 
 perform :: Key -> FilePath -> CommandPerform
 perform key file = do
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index b37a26e122..9383c07f27 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -584,7 +584,7 @@ prepIncremental u (Just StartIncrementalO) = do
 	recordStartTime u
 	ifM (FsckDb.newPass u)
 		( StartIncremental <$> openFsckDb u
-		, error "Cannot start a new --incremental fsck pass; another fsck process is already running."
+		, giveup "Cannot start a new --incremental fsck pass; another fsck process is already running."
 		)
 prepIncremental u (Just MoreIncrementalO) =
 	ContIncremental <$> openFsckDb u
diff --git a/Command/FuzzTest.hs b/Command/FuzzTest.hs
index 4aed02d465..0c5aac9b33 100644
--- a/Command/FuzzTest.hs
+++ b/Command/FuzzTest.hs
@@ -39,7 +39,7 @@ start = do
 
 guardTest :: Annex ()
 guardTest = unlessM (fromMaybe False . Git.Config.isTrue <$> getConfig key "") $
-	error $ unlines
+	giveup $ unlines
 		[ "Running fuzz tests *writes* to and *deletes* files in"
 		, "this repository, and pushes those changes to other"
 		, "repositories! This is a developer tool, not something"
diff --git a/Command/GCryptSetup.hs b/Command/GCryptSetup.hs
index f2943ea134..cbc2de0efb 100644
--- a/Command/GCryptSetup.hs
+++ b/Command/GCryptSetup.hs
@@ -25,7 +25,7 @@ start :: String -> CommandStart
 start gcryptid = next $ next $ do
 	u <- getUUID
 	when (u /= NoUUID) $
-		error "gcryptsetup refusing to run; this repository already has a git-annex uuid!"
+		giveup "gcryptsetup refusing to run; this repository already has a git-annex uuid!"
 	
 	g <- gitRepo
 	gu <- Remote.GCrypt.getGCryptUUID True g
@@ -35,5 +35,5 @@ start gcryptid = next $ next $ do
 			then do
 				void $ Remote.GCrypt.setupRepo gcryptid g
 				return True
-			else error "cannot use gcrypt in a non-bare repository"
-		else error "gcryptsetup uuid mismatch"
+			else giveup "cannot use gcrypt in a non-bare repository"
+		else giveup "gcryptsetup uuid mismatch"
diff --git a/Command/Group.hs b/Command/Group.hs
index 8e901dfb34..6d9b4ab131 100644
--- a/Command/Group.hs
+++ b/Command/Group.hs
@@ -30,7 +30,7 @@ start (name:[]) = do
 	u <- Remote.nameToUUID name
 	showRaw . unwords . S.toList =<< lookupGroups u
 	stop
-start _ = error "Specify a repository and a group."
+start _ = giveup "Specify a repository and a group."
 
 setGroup :: UUID -> Group -> CommandPerform
 setGroup uuid g = do
diff --git a/Command/GroupWanted.hs b/Command/GroupWanted.hs
index 6a9e300bf3..c0be2462db 100644
--- a/Command/GroupWanted.hs
+++ b/Command/GroupWanted.hs
@@ -25,4 +25,4 @@ start (g:[]) = next $ performGet groupPreferredContentMapRaw g
 start (g:expr:[]) = do
 	showStart "groupwanted" g
 	next $ performSet groupPreferredContentSet expr g
-start _ = error "Specify a group."
+start _ = giveup "Specify a group."
diff --git a/Command/Import.hs b/Command/Import.hs
index d5a2feed59..a16349ad2a 100644
--- a/Command/Import.hs
+++ b/Command/Import.hs
@@ -62,7 +62,7 @@ seek o = allowConcurrentOutput $ do
 	repopath <- liftIO . absPath =<< fromRepo Git.repoPath
 	inrepops <- liftIO $ filter (dirContains repopath) <$> mapM absPath (importFiles o)
 	unless (null inrepops) $ do
-		error $ "cannot import files from inside the working tree (use git annex add instead): " ++ unwords inrepops
+		giveup $ "cannot import files from inside the working tree (use git annex add instead): " ++ unwords inrepops
 	largematcher <- largeFilesMatcher
 	withPathContents (start largematcher (duplicateMode o)) (importFiles o)
 
diff --git a/Command/ImportFeed.hs b/Command/ImportFeed.hs
index 8f3a607261..1736f2567c 100644
--- a/Command/ImportFeed.hs
+++ b/Command/ImportFeed.hs
@@ -147,7 +147,7 @@ findDownloads u = go =<< downloadFeed u
 {- Feeds change, so a feed download cannot be resumed. -}
 downloadFeed :: URLString -> Annex (Maybe Feed)
 downloadFeed url
-	| Url.parseURIRelaxed url == Nothing = error "invalid feed url"
+	| Url.parseURIRelaxed url == Nothing = giveup "invalid feed url"
 	| otherwise = do
 		showOutput
 		uo <- Url.getUrlOptions
@@ -336,7 +336,7 @@ noneValue = "none"
  - Throws an error if the feed is broken, otherwise shows a warning. -}
 feedProblem :: URLString -> String -> Annex ()
 feedProblem url message = ifM (checkFeedBroken url)
-	( error $ message ++ " (having repeated problems with feed: " ++ url ++ ")"
+	( giveup $ message ++ " (having repeated problems with feed: " ++ url ++ ")"
 	, warning $ "warning: " ++ message
 	)
 
diff --git a/Command/Indirect.hs b/Command/Indirect.hs
index 74841a5f63..f12f9e59e7 100644
--- a/Command/Indirect.hs
+++ b/Command/Indirect.hs
@@ -33,9 +33,9 @@ start :: CommandStart
 start = ifM isDirect
 	( do
 		unlessM (coreSymlinks <$> Annex.getGitConfig) $
-			error "Git is configured to not use symlinks, so you must use direct mode."
+			giveup "Git is configured to not use symlinks, so you must use direct mode."
 		whenM probeCrippledFileSystem $
-			error "This repository seems to be on a crippled filesystem, you must use direct mode."
+			giveup "This repository seems to be on a crippled filesystem, you must use direct mode."
 		next perform
 	, stop
 	)
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 05717bc609..e5d7a90390 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -26,16 +26,16 @@ seek :: CmdParams -> CommandSeek
 seek = withWords start
 
 start :: [String] -> CommandStart
-start [] = error "Specify a name for the remote."
+start [] = giveup "Specify a name for the remote."
 start (name:ws) = ifM (isJust <$> findExisting name)
-	( error $ "There is already a special remote named \"" ++ name ++
+	( giveup $ "There is already a special remote named \"" ++ name ++
 		"\". (Use enableremote to enable an existing special remote.)"
 	, do
 		ifM (isJust <$> Remote.byNameOnly name)
-			( error $ "There is already a remote named \"" ++ name ++ "\""
+			( giveup $ "There is already a remote named \"" ++ name ++ "\""
 			, do
 				let c = newConfig name
-				t <- either error return (findType config)
+				t <- either giveup return (findType config)
 
 				showStart "initremote" name
 				next $ perform t name $ M.union config c
diff --git a/Command/Lock.hs b/Command/Lock.hs
index 68360705cc..a3fc251172 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -79,7 +79,7 @@ performNew file key = do
 		unlessM (sameInodeCache obj (maybeToList mfc)) $ do
 			modifyContent obj $ replaceFile obj $ \tmp -> do
 				unlessM (checkedCopyFile key obj tmp Nothing) $
-					error "unable to lock file"
+					giveup "unable to lock file"
 			Database.Keys.storeInodeCaches key [obj]
 
 	-- Try to repopulate obj from an unmodified associated file.
@@ -115,4 +115,4 @@ performOld file = do
 	next $ return True
 
 errorModified :: a
-errorModified =  error "Locking this file would discard any changes you have made to it. Use 'git annex add' to stage your changes. (Or, use --force to override)"
+errorModified =  giveup "Locking this file would discard any changes you have made to it. Use 'git annex add' to stage your changes. (Or, use --force to override)"
diff --git a/Command/LockContent.hs b/Command/LockContent.hs
index de697c0901..35342c529b 100644
--- a/Command/LockContent.hs
+++ b/Command/LockContent.hs
@@ -32,7 +32,7 @@ start [ks] = do
 		then exitSuccess
 		else exitFailure
   where
-	k = fromMaybe (error "bad key") (file2key ks)
+	k = fromMaybe (giveup "bad key") (file2key ks)
 	locksuccess = ifM (inAnnex k)
 		( liftIO $ do
 			putStrLn contentLockedMarker
@@ -41,4 +41,4 @@ start [ks] = do
 			return True
 		, return False
 		)
-start _ = error "Specify exactly 1 key."
+start _ = giveup "Specify exactly 1 key."
diff --git a/Command/Log.hs b/Command/Log.hs
index 3806d8fdf7..357bcf1f3e 100644
--- a/Command/Log.hs
+++ b/Command/Log.hs
@@ -93,7 +93,7 @@ seek o = do
 	case (logFiles o, allOption o) of
 		(fs, False) -> withFilesInGit (whenAnnexed $ start o outputter) fs
 		([], True) -> commandAction (startAll o outputter)
-		(_, True) -> error "Cannot specify both files and --all"
+		(_, True) -> giveup "Cannot specify both files and --all"
 
 start :: LogOptions -> (FilePath -> Outputter) -> FilePath -> Key -> CommandStart
 start o outputter file key = do
diff --git a/Command/MetaData.hs b/Command/MetaData.hs
index 6e64207c83..04d859e4cf 100644
--- a/Command/MetaData.hs
+++ b/Command/MetaData.hs
@@ -81,7 +81,7 @@ seek o = do
 		Batch -> withMessageState $ \s -> case outputType s of
 			JSONOutput _ -> batchInput parseJSONInput $
 				commandAction . startBatch now
-			_ -> error "--batch is currently only supported in --json mode"
+			_ -> giveup "--batch is currently only supported in --json mode"
 
 start :: POSIXTime -> MetaDataOptions -> FilePath -> Key -> CommandStart
 start now o file k = startKeys now o k (mkActionItem afile)
@@ -156,7 +156,7 @@ startBatch now (i, (MetaData m)) = case i of
 		mk <- lookupFile f
 		case mk of
 			Just k -> go k (mkActionItem (Just f))
-			Nothing -> error $ "not an annexed file: " ++ f
+			Nothing -> giveup $ "not an annexed file: " ++ f
 	Right k -> go k (mkActionItem k)
   where
 	go k ai = do
diff --git a/Command/Move.hs b/Command/Move.hs
index 9c43c6f1db..d74eea9003 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -197,4 +197,4 @@ fromPerform src move key afile = ifM (inAnnex key)
 			]
 		ok <- Remote.removeKey src key
 		next $ Command.Drop.cleanupRemote key src ok
-	faileddropremote = error "Unable to drop from remote."
+	faileddropremote = giveup "Unable to drop from remote."
diff --git a/Command/NumCopies.hs b/Command/NumCopies.hs
index 0a9c4404b4..005a0d16af 100644
--- a/Command/NumCopies.hs
+++ b/Command/NumCopies.hs
@@ -23,15 +23,15 @@ seek = withWords start
 start :: [String] -> CommandStart
 start [] = startGet
 start [s] = case readish s of
-	Nothing -> error $ "Bad number: " ++ s
+	Nothing -> giveup $ "Bad number: " ++ s
 	Just n
 		| n > 0 -> startSet n
 		| n == 0 -> ifM (Annex.getState Annex.force)
 			( startSet n
-			, error "Setting numcopies to 0 is very unsafe. You will lose data! If you really want to do that, specify --force."
+			, giveup "Setting numcopies to 0 is very unsafe. You will lose data! If you really want to do that, specify --force."
 			)
-		| otherwise -> error "Number cannot be negative!"
-start _ = error "Specify a single number."
+		| otherwise -> giveup "Number cannot be negative!"
+start _ = giveup "Specify a single number."
 
 startGet :: CommandStart
 startGet = next $ next $ do
diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs
index f55318475c..1ff2227d83 100644
--- a/Command/PreCommit.hs
+++ b/Command/PreCommit.hs
@@ -46,7 +46,7 @@ seek ps = lockPreCommitHook $ ifM isDirect
 			( do
 				(fs, cleanup) <- inRepo $ Git.typeChangedStaged ps
 				whenM (anyM isOldUnlocked fs) $
-					error "Cannot make a partial commit with unlocked annexed files. You should `git annex add` the files you want to commit, and then run git commit."
+					giveup "Cannot make a partial commit with unlocked annexed files. You should `git annex add` the files you want to commit, and then run git commit."
 				void $ liftIO cleanup
 			, do
 				-- fix symlinks to files being committed
diff --git a/Command/Proxy.hs b/Command/Proxy.hs
index f1f7f194f6..dba0300b8c 100644
--- a/Command/Proxy.hs
+++ b/Command/Proxy.hs
@@ -30,7 +30,7 @@ seek :: CmdParams -> CommandSeek
 seek = withWords start
 
 start :: [String] -> CommandStart
-start [] = error "Did not specify command to run."
+start [] = giveup "Did not specify command to run."
 start (c:ps) = liftIO . exitWith =<< ifM isDirect
 	( do
 		tmp <- gitAnnexTmpMiscDir <$> gitRepo
diff --git a/Command/ReKey.hs b/Command/ReKey.hs
index 4d20395307..51f9f6fe15 100644
--- a/Command/ReKey.hs
+++ b/Command/ReKey.hs
@@ -33,7 +33,7 @@ seek = withPairs start
 start :: (FilePath, String) -> CommandStart
 start (file, keyname) = ifAnnexed file go stop
   where
-	newkey = fromMaybe (error "bad key") $ file2key keyname
+	newkey = fromMaybe (giveup "bad key") $ file2key keyname
 	go oldkey
 		| oldkey == newkey = stop
 		| otherwise = do
@@ -46,7 +46,7 @@ perform file oldkey newkey = do
 		( unlessM (linkKey file oldkey newkey) $
 			error "failed"
 		, unlessM (Annex.getState Annex.force) $
-			error $ file ++ " is not available (use --force to override)"
+			giveup $ file ++ " is not available (use --force to override)"
 		)
 	next $ cleanup file oldkey newkey
 
diff --git a/Command/ReadPresentKey.hs b/Command/ReadPresentKey.hs
index 1eba2cc122..f73e22af40 100644
--- a/Command/ReadPresentKey.hs
+++ b/Command/ReadPresentKey.hs
@@ -27,5 +27,5 @@ start (ks:us:[]) = do
 		then liftIO exitSuccess
 		else liftIO exitFailure
   where
-	k = fromMaybe (error "bad key") (file2key ks)
-start _ = error "Wrong number of parameters"
+	k = fromMaybe (giveup "bad key") (file2key ks)
+start _ = giveup "Wrong number of parameters"
diff --git a/Command/RegisterUrl.hs b/Command/RegisterUrl.hs
index 273d111b0c..28dd2d8c5a 100644
--- a/Command/RegisterUrl.hs
+++ b/Command/RegisterUrl.hs
@@ -32,7 +32,7 @@ start (keyname:url:[]) = do
 start [] = do
 	showStart "registerurl" "stdin"
 	next massAdd
-start _ = error "specify a key and an url"
+start _ = giveup "specify a key and an url"
 
 massAdd :: CommandPerform
 massAdd = go True =<< map (separate (== ' ')) . lines <$> liftIO getContents
@@ -43,7 +43,7 @@ massAdd = go True =<< map (separate (== ' ')) . lines <$> liftIO getContents
 		ok <- perform' key u
 		let !status' = status && ok
 		go status' rest
-	go _ _ = error "Expected pairs of key and url on stdin, but got something else."
+	go _ _ = giveup "Expected pairs of key and url on stdin, but got something else."
 
 perform :: Key -> URLString -> CommandPerform
 perform key url = do
diff --git a/Command/Reinject.hs b/Command/Reinject.hs
index fa2459e22d..97aa602e7b 100644
--- a/Command/Reinject.hs
+++ b/Command/Reinject.hs
@@ -47,7 +47,7 @@ startSrcDest (src:dest:[])
 		next $ ifAnnexed dest
 			(\key -> perform src key (verifyKeyContent DefaultVerify UnVerified key src))
 			stop
-startSrcDest _ = error "specify a src file and a dest file"
+startSrcDest _ = giveup "specify a src file and a dest file"
 
 startKnown :: FilePath -> CommandStart
 startKnown src = notAnnexed src $ do
@@ -63,7 +63,8 @@ startKnown src = notAnnexed src $ do
 			)
 
 notAnnexed :: FilePath -> CommandStart -> CommandStart
-notAnnexed src = ifAnnexed src (error $ "cannot used annexed file as src: " ++ src)
+notAnnexed src = ifAnnexed src $
+	giveup $ "cannot used annexed file as src: " ++ src
 
 perform :: FilePath -> Key -> Annex Bool -> CommandPerform
 perform src key verify = ifM move
diff --git a/Command/ResolveMerge.hs b/Command/ResolveMerge.hs
index 8742a1104f..0ba6efb362 100644
--- a/Command/ResolveMerge.hs
+++ b/Command/ResolveMerge.hs
@@ -33,8 +33,8 @@ start = do
 		( do
 			void $ commitResolvedMerge Git.Branch.ManualCommit
 			next $ next $ return True
-		, error "Merge conflict could not be automatically resolved."
+		, giveup "Merge conflict could not be automatically resolved."
 		)
   where
-	nobranch = error "No branch is currently checked out."
-	nomergehead = error "No SHA found in .git/merge_head"
+	nobranch = giveup "No branch is currently checked out."
+	nomergehead = giveup "No SHA found in .git/merge_head"
diff --git a/Command/Schedule.hs b/Command/Schedule.hs
index 5721e98e76..5cc8b37bf5 100644
--- a/Command/Schedule.hs
+++ b/Command/Schedule.hs
@@ -31,7 +31,7 @@ start = parse
 	parse (name:expr:[]) = go name $ \uuid -> do
 		showStart "schedile" name
 		performSet expr uuid
-	parse _ = error "Specify a repository."
+	parse _ = giveup "Specify a repository."
 
 	go name a = do
 		u <- Remote.nameToUUID name
@@ -47,7 +47,7 @@ performGet uuid = do
 
 performSet :: String -> UUID -> CommandPerform
 performSet expr uuid = case parseScheduledActivities expr of
-	Left e -> error $ "Parse error: " ++ e
+	Left e -> giveup $ "Parse error: " ++ e
 	Right l -> do
 		scheduleSet uuid l
 		next $ return True
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index fd7a4ab887..090edee0ba 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -23,10 +23,10 @@ start :: [String] -> CommandStart
 start (keyname:file:[]) = do
 	showStart "setkey" file
 	next $ perform file (mkKey keyname)
-start _ = error "specify a key and a content file"
+start _ = giveup "specify a key and a content file"
 
 mkKey :: String -> Key
-mkKey = fromMaybe (error "bad key") . file2key
+mkKey = fromMaybe (giveup "bad key") . file2key
 
 perform :: FilePath -> Key -> CommandPerform
 perform file key = do
diff --git a/Command/SetPresentKey.hs b/Command/SetPresentKey.hs
index 20c96ae365..da2a6fa3d9 100644
--- a/Command/SetPresentKey.hs
+++ b/Command/SetPresentKey.hs
@@ -26,9 +26,9 @@ start (ks:us:vs:[]) = do
 	showStart' "setpresentkey" k (mkActionItem k)
 	next $ perform k (toUUID us) s
   where
-	k = fromMaybe (error "bad key") (file2key ks)
-	s = fromMaybe (error "bad value") (parseStatus vs)
-start _ = error "Wrong number of parameters"
+	k = fromMaybe (giveup "bad key") (file2key ks)
+	s = fromMaybe (giveup "bad value") (parseStatus vs)
+start _ = giveup "Wrong number of parameters"
 
 perform :: Key -> UUID -> LogStatus -> CommandPerform
 perform k u s = next $ do
diff --git a/Command/Sync.hs b/Command/Sync.hs
index fb80c3e74b..acc5fbbc94 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -292,7 +292,7 @@ updateSyncBranch (Just branch, madj) = do
 
 updateBranch :: Git.Branch -> Git.Branch -> Git.Repo -> IO ()
 updateBranch syncbranch updateto g = 
-	unlessM go $ error $ "failed to update " ++ Git.fromRef syncbranch
+	unlessM go $ giveup $ "failed to update " ++ Git.fromRef syncbranch
   where
 	go = Git.Command.runBool
 		[ Param "branch"
diff --git a/Command/TestRemote.hs b/Command/TestRemote.hs
index 40d02c166c..4c0ff9e3c8 100644
--- a/Command/TestRemote.hs
+++ b/Command/TestRemote.hs
@@ -57,7 +57,7 @@ seek o = commandAction $ start (fromInteger $ sizeOption o) (testRemote o)
 start :: Int -> RemoteName -> CommandStart
 start basesz name = do
 	showStart "testremote" name
-	r <- either error id <$> Remote.byName' name
+	r <- either giveup id <$> Remote.byName' name
 	showAction "generating test keys"
 	fast <- Annex.getState Annex.fast
 	ks <- mapM randKey (keySizes basesz fast)
diff --git a/Command/TransferInfo.hs b/Command/TransferInfo.hs
index 21b7830c31..6870c84f01 100644
--- a/Command/TransferInfo.hs
+++ b/Command/TransferInfo.hs
@@ -59,7 +59,7 @@ start (k:[]) = do
 				, exitSuccess
 				]
 	stop
-start _ = error "wrong number of parameters"
+start _ = giveup "wrong number of parameters"
 
 readUpdate :: IO (Maybe Integer)
 readUpdate = readish <$> getLine
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 4e83fd4209..e744b51a86 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -45,7 +45,7 @@ wrapUnannex a = ifM (versionSupportsUnlockedPointers <||> isDirect)
 	 -}
 	, ifM cleanindex
 		( lockPreCommitHook $ commit `after` a
-		, error "Cannot proceed with uncommitted changes staged in the index. Recommend you: git commit"
+		, giveup "Cannot proceed with uncommitted changes staged in the index. Recommend you: git commit"
 		)
 	)
   where
diff --git a/Command/Undo.hs b/Command/Undo.hs
index 24c099f92c..c366453a3d 100644
--- a/Command/Undo.hs
+++ b/Command/Undo.hs
@@ -32,7 +32,7 @@ seek ps = do
 	-- in the index.
 	(fs, cleanup) <- inRepo $ LsFiles.notInRepo False ps
 	unless (null fs) $
-		error $ "Cannot undo changes to files that are not checked into git: " ++ unwords fs
+		giveup $ "Cannot undo changes to files that are not checked into git: " ++ unwords fs
 	void $ liftIO $ cleanup
 
 	-- Committing staged changes before undo allows later
diff --git a/Command/Ungroup.hs b/Command/Ungroup.hs
index 5f84a375f0..ddcdba4667 100644
--- a/Command/Ungroup.hs
+++ b/Command/Ungroup.hs
@@ -26,7 +26,7 @@ start (name:g:[]) = do
 	showStart "ungroup" name
 	u <- Remote.nameToUUID name
 	next $ perform u g
-start _ = error "Specify a repository and a group."
+start _ = giveup "Specify a repository and a group."
 
 perform :: UUID -> Group -> CommandPerform
 perform uuid g = do
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index fa7e130137..d8c7d1295a 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -30,12 +30,12 @@ cmd = addCheck check $
 check :: Annex ()
 check = do
 	b <- current_branch
-	when (b == Annex.Branch.name) $ error $
+	when (b == Annex.Branch.name) $ giveup $
 		"cannot uninit when the " ++ Git.fromRef b ++ " branch is checked out"
 	top <- fromRepo Git.repoPath
 	currdir <- liftIO getCurrentDirectory
 	whenM ((/=) <$> liftIO (absPath top) <*> liftIO (absPath currdir)) $
-		error "can only run uninit from the top of the git repository"
+		giveup "can only run uninit from the top of the git repository"
   where
 	current_branch = Git.Ref . Prelude.head . lines <$> revhead
 	revhead = inRepo $ Git.Command.pipeReadStrict
@@ -51,7 +51,7 @@ seek ps = do
 {- git annex symlinks that are not checked into git could be left by an
  - interrupted add. -}
 startCheckIncomplete :: FilePath -> Key -> CommandStart
-startCheckIncomplete file _ = error $ unlines
+startCheckIncomplete file _ = giveup $ unlines
 	[ file ++ " points to annexed content, but is not checked into git."
 	, "Perhaps this was left behind by an interrupted git annex add?"
 	, "Not continuing with uninit; either delete or git annex add the file and retry."
@@ -65,7 +65,7 @@ finish = do
 	prepareRemoveAnnexDir annexdir
 	if null leftovers
 		then liftIO $ removeDirectoryRecursive annexdir
-		else error $ unlines
+		else giveup $ unlines
 			[ "Not fully uninitialized"
 			, "Some annexed data is still left in " ++ annexobjectdir
 			, "This may include deleted files, or old versions of modified files."
diff --git a/Command/Unused.hs b/Command/Unused.hs
index c116cdc0e1..1711fe047c 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -320,7 +320,7 @@ unusedSpec m spec
 	range (a, b) = case (readish a, readish b) of
 		(Just x, Just y) -> [x..y]
 		_ -> badspec
-	badspec = error $ "Expected number or range, not \"" ++ spec ++ "\""
+	badspec = giveup $ "Expected number or range, not \"" ++ spec ++ "\""
 
 {- Seek action for unused content. Finds the number in the maps, and
  - calls one of 3 actions, depending on the type of unused file. -}
@@ -335,7 +335,7 @@ startUnused message unused badunused tmpunused maps n = search
 	, (unusedTmpMap maps, tmpunused)
 	]
   where
-	search [] = error $ show n ++ " not valid (run git annex unused for list)"
+	search [] = giveup $ show n ++ " not valid (run git annex unused for list)"
 	search ((m, a):rest) =
 		case M.lookup n m of
 			Nothing -> search rest
diff --git a/Command/VAdd.hs b/Command/VAdd.hs
index a4b3f379f2..c94ce57229 100644
--- a/Command/VAdd.hs
+++ b/Command/VAdd.hs
@@ -33,6 +33,6 @@ start params = do
 				next $ next $ return True
 			Narrowing -> next $ next $ do
 				if visibleViewSize view' == visibleViewSize view
-					then error "That would not add an additional level of directory structure to the view. To filter the view, use vfilter instead of vadd."
+					then giveup "That would not add an additional level of directory structure to the view. To filter the view, use vfilter instead of vadd."
 					else checkoutViewBranch view' narrowView
-			Widening -> error "Widening view to match more files is not currently supported."
+			Widening -> giveup "Widening view to match more files is not currently supported."
diff --git a/Command/VCycle.hs b/Command/VCycle.hs
index 20fc9a22a6..28326e16fa 100644
--- a/Command/VCycle.hs
+++ b/Command/VCycle.hs
@@ -25,7 +25,7 @@ seek = withNothing start
 start ::CommandStart
 start = go =<< currentView
   where
-	go Nothing = error "Not in a view."
+	go Nothing = giveup "Not in a view."
 	go (Just v) = do
 		showStart "vcycle" ""
 		let v' = v { viewComponents = vcycle [] (viewComponents v) }
diff --git a/Command/VFilter.hs b/Command/VFilter.hs
index 60bbcd3d3f..130e2550cb 100644
--- a/Command/VFilter.hs
+++ b/Command/VFilter.hs
@@ -26,5 +26,5 @@ start params = do
 		let view' = filterView view $
 			map parseViewParam $ reverse params
 		next $ next $ if visibleViewSize view' > visibleViewSize view
-			then error "That would add an additional level of directory structure to the view, rather than filtering it. If you want to do that, use vadd instead of vfilter."
+			then giveup "That would add an additional level of directory structure to the view, rather than filtering it. If you want to do that, use vadd instead of vfilter."
 			else checkoutViewBranch view' narrowView
diff --git a/Command/VPop.hs b/Command/VPop.hs
index 8490567dc6..58411001be 100644
--- a/Command/VPop.hs
+++ b/Command/VPop.hs
@@ -26,7 +26,7 @@ seek = withWords start
 start :: [String] -> CommandStart
 start ps = go =<< currentView
   where
-	go Nothing = error "Not in a view."
+	go Nothing = giveup "Not in a view."
 	go (Just v) = do
 		showStart "vpop" (show num)
 		removeView v
diff --git a/Command/Vicfg.hs b/Command/Vicfg.hs
index d7963725a9..64daa598b7 100644
--- a/Command/Vicfg.hs
+++ b/Command/Vicfg.hs
@@ -50,7 +50,7 @@ vicfg curcfg f = do
 	vi <- liftIO $ catchDefaultIO "vi" $ getEnv "EDITOR"
 	-- Allow EDITOR to be processed by the shell, so it can contain options.
 	unlessM (liftIO $ boolSystem "sh" [Param "-c", Param $ unwords [vi, shellEscape f]]) $
-		error $ vi ++ " exited nonzero; aborting"
+		giveup $ vi ++ " exited nonzero; aborting"
 	r <- parseCfg (defCfg curcfg) <$> liftIO (readFileStrictAnyEncoding f)
 	liftIO $ nukeFile f
 	case r of
diff --git a/Command/View.hs b/Command/View.hs
index 65985fdac9..513e6d10c0 100644
--- a/Command/View.hs
+++ b/Command/View.hs
@@ -25,7 +25,7 @@ seek :: CmdParams -> CommandSeek
 seek = withWords start
 
 start :: [String] -> CommandStart
-start [] = error "Specify metadata to include in view"
+start [] = giveup "Specify metadata to include in view"
 start ps = do
 	showStart "view" ""
 	view <- mkView ps
@@ -34,7 +34,7 @@ start ps = do
 	go view Nothing = next $ perform view
 	go view (Just v)
 		| v == view = stop
-		| otherwise = error "Already in a view. Use the vfilter and vadd commands to further refine this view."
+		| otherwise = giveup "Already in a view. Use the vfilter and vadd commands to further refine this view."
 
 perform :: View -> CommandPerform
 perform view = do
@@ -47,7 +47,7 @@ paramView = paramRepeating "FIELD=VALUE"
 mkView :: [String] -> Annex View
 mkView ps = go =<< inRepo Git.Branch.current
   where
-	go Nothing = error "not on any branch!"
+	go Nothing = giveup "not on any branch!"
 	go (Just b) = return $ fst $ refineView (View b []) $
 		map parseViewParam $ reverse ps
 
diff --git a/Command/Wanted.hs b/Command/Wanted.hs
index dca92a7b42..8fd369df6a 100644
--- a/Command/Wanted.hs
+++ b/Command/Wanted.hs
@@ -37,7 +37,7 @@ cmd' name desc getter setter = command name SectionSetup desc pdesc (withParams
 	start (rname:expr:[]) = go rname $ \uuid -> do
 		showStart name rname
 		performSet setter expr uuid
-	start _ = error "Specify a repository."
+	start _ = giveup "Specify a repository."
 		
 	go rname a = do
 		u <- Remote.nameToUUID rname
@@ -52,7 +52,7 @@ performGet getter a = do
 
 performSet :: (a -> PreferredContentExpression -> Annex ()) -> String -> a -> CommandPerform
 performSet setter expr a = case checkPreferredContentExpression expr of
-	Just e -> error $ "Parse error: " ++ e
+	Just e -> giveup $ "Parse error: " ++ e
 	Nothing -> do
 		setter a expr
 		next $ return True
diff --git a/Command/WebApp.hs b/Command/WebApp.hs
index 4dff8c9d16..d9c001b226 100644
--- a/Command/WebApp.hs
+++ b/Command/WebApp.hs
@@ -77,7 +77,7 @@ start' allowauto o = do
 			else annexListen <$> Annex.getGitConfig
 		ifM (checkpid <&&> checkshim f)
 			( if isJust (listenAddress o)
-				then error "The assistant is already running, so --listen cannot be used."
+				then giveup "The assistant is already running, so --listen cannot be used."
 				else do
 					url <- liftIO . readFile
 						=<< fromRepo gitAnnexUrlFile
@@ -125,7 +125,7 @@ startNoRepo o = go =<< liftIO (filterM doesDirectoryExist =<< readAutoStartFile)
 				go ds
 			Right state -> void $ Annex.eval state $ do
 				whenM (fromRepo Git.repoIsLocalBare) $
-					error $ d ++ " is a bare git repository, cannot run the webapp in it"
+					giveup $ d ++ " is a bare git repository, cannot run the webapp in it"
 				callCommandAction $
 					start' False o
 
diff --git a/Config/Files.hs b/Config/Files.hs
index 8f8b4c1158..b18d912e9e 100644
--- a/Config/Files.hs
+++ b/Config/Files.hs
@@ -80,4 +80,4 @@ readProgramFile = do
 cannotFindProgram :: IO a
 cannotFindProgram = do
 	f <- programFile
-	error $ "cannot find git-annex program in PATH or in the location listed in " ++ f
+	giveup $ "cannot find git-annex program in PATH or in the location listed in " ++ f
diff --git a/Creds.hs b/Creds.hs
index e818317c78..6be9b33916 100644
--- a/Creds.hs
+++ b/Creds.hs
@@ -105,7 +105,7 @@ getRemoteCredPair c gc storage = maybe fromcache (return . Just) =<< fromenv
 				-- Not a problem for shared cipher.
 				case storablecipher of
 					SharedCipher {} -> showLongNote "gpg error above was caused by an old git-annex bug in credentials storage. Working around it.."
-					_ -> error "*** Insecure credentials storage detected for this remote! See https://git-annex.branchable.com/upgrades/insecure_embedded_creds/"
+					_ -> giveup "*** Insecure credentials storage detected for this remote! See https://git-annex.branchable.com/upgrades/insecure_embedded_creds/"
 				fromcreds $ fromB64 enccreds
 	fromcreds creds = case decodeCredPair creds of
 		Just credpair -> do
diff --git a/Crypto.hs b/Crypto.hs
index f3d6f5e5a9..d3cbfa2f7f 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -100,7 +100,7 @@ genSharedPubKeyCipher cmd keyid highQuality = do
  -
  - When the Cipher is encrypted, re-encrypts it. -}
 updateCipherKeyIds :: LensGpgEncParams encparams => Gpg.GpgCmd -> encparams -> [(Bool, Gpg.KeyId)] -> StorableCipher -> IO StorableCipher
-updateCipherKeyIds _ _ _ SharedCipher{} = error "Cannot update shared cipher"
+updateCipherKeyIds _ _ _ SharedCipher{} = giveup "Cannot update shared cipher"
 updateCipherKeyIds _ _ [] c = return c
 updateCipherKeyIds cmd encparams changes encipher@(EncryptedCipher _ variant ks) = do
 	ks' <- updateCipherKeyIds' cmd changes ks
@@ -113,11 +113,11 @@ updateCipherKeyIds' :: Gpg.GpgCmd -> [(Bool, Gpg.KeyId)] -> KeyIds -> IO KeyIds
 updateCipherKeyIds' cmd changes (KeyIds ks) = do
 	dropkeys <- listKeyIds [ k | (False, k) <- changes ]
 	forM_ dropkeys $ \k -> unless (k `elem` ks) $
-		error $ "Key " ++ k ++ " was not present; cannot remove."
+		giveup $ "Key " ++ k ++ " was not present; cannot remove."
 	addkeys <- listKeyIds [ k | (True, k) <- changes ]
 	let ks' = (addkeys ++ ks) \\ dropkeys
 	when (null ks') $
-		error "Cannot remove the last key."
+		giveup "Cannot remove the last key."
 	return $ KeyIds ks'
   where
 	listKeyIds = concat <$$> mapM (keyIds <$$> Gpg.findPubKeys cmd)
diff --git a/Database/Types.hs b/Database/Types.hs
index 4521bb3463..9eabc6983e 100644
--- a/Database/Types.hs
+++ b/Database/Types.hs
@@ -25,7 +25,7 @@ toSKey :: Key -> SKey
 toSKey = SKey . key2file
 
 fromSKey :: SKey -> Key
-fromSKey (SKey s) = fromMaybe (error $ "bad serialied Key " ++ s) (file2key s)
+fromSKey (SKey s) = fromMaybe (error $ "bad serialized Key " ++ s) (file2key s)
 
 derivePersistField "SKey"
 
@@ -43,7 +43,7 @@ toIKey :: Key -> IKey
 toIKey = IKey . key2file
 
 fromIKey :: IKey -> Key
-fromIKey (IKey s) = fromMaybe (error $ "bad serialied Key " ++ s) (file2key s)
+fromIKey (IKey s) = fromMaybe (error $ "bad serialized Key " ++ s) (file2key s)
 
 derivePersistField "IKey"
 
diff --git a/Git/AutoCorrect.hs b/Git/AutoCorrect.hs
index 7a9d788511..ae7cc91a87 100644
--- a/Git/AutoCorrect.hs
+++ b/Git/AutoCorrect.hs
@@ -50,7 +50,7 @@ prepare input showmatch matches r =
 			| otherwise -> sleep n
 		Nothing -> list
   where
-	list = error $ unlines $
+	list = giveup $ unlines $
 		[ "Unknown command '" ++ input ++ "'"
 		, ""
 		, "Did you mean one of these?"
diff --git a/Git/CurrentRepo.hs b/Git/CurrentRepo.hs
index dab4ad21bf..69a679ee36 100644
--- a/Git/CurrentRepo.hs
+++ b/Git/CurrentRepo.hs
@@ -52,7 +52,7 @@ get = do
 		curr <- getCurrentDirectory
 		Git.Config.read $ newFrom $
 			Local { gitdir = absd, worktree = Just curr }
-	configure Nothing Nothing = error "Not in a git repository."
+	configure Nothing Nothing = giveup "Not in a git repository."
 
 	addworktree w r = changelocation r $
 		Local { gitdir = gitdir (location r), worktree = w }
diff --git a/Git/GCrypt.hs b/Git/GCrypt.hs
index 2a2f7dfe15..e61b763588 100644
--- a/Git/GCrypt.hs
+++ b/Git/GCrypt.hs
@@ -46,7 +46,7 @@ encryptedRemote baserepo = go
 		u = show url
 		plen = length urlPrefix
 	go _ = notencrypted
-	notencrypted = error "not a gcrypt encrypted repository"
+	notencrypted = giveup "not a gcrypt encrypted repository"
 
 data ProbeResult = Decryptable | NotDecryptable | NotEncrypted
 
diff --git a/Limit.hs b/Limit.hs
index 4bd5dd59e3..efe4fea85b 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -73,7 +73,7 @@ addToken = add . Utility.Matcher.token
 
 {- Adds a new limit. -}
 addLimit :: Either String (MatchFiles Annex) -> Annex ()
-addLimit = either error (\l -> add $ Utility.Matcher.Operation $ l S.empty)
+addLimit = either giveup (\l -> add $ Utility.Matcher.Operation $ l S.empty)
 
 {- Add a limit to skip files that do not match the glob. -}
 addInclude :: String -> Annex ()
@@ -289,7 +289,7 @@ limitMetaData s = case parseMetaDataMatcher s of
 
 addTimeLimit :: String -> Annex ()
 addTimeLimit s = do
-	let seconds = maybe (error "bad time-limit") durationToPOSIXTime $
+	let seconds = maybe (giveup "bad time-limit") durationToPOSIXTime $
 		parseDuration s
 	start <- liftIO getPOSIXTime
 	let cutoff = start + seconds
diff --git a/Logs/Transitions.hs b/Logs/Transitions.hs
index 07667c4074..04f9824b16 100644
--- a/Logs/Transitions.hs
+++ b/Logs/Transitions.hs
@@ -60,7 +60,7 @@ parseTransitions = check . map parseTransitionLine . splitLines
 parseTransitionsStrictly :: String -> String -> Transitions
 parseTransitionsStrictly source = fromMaybe badsource . parseTransitions
   where
-	badsource = error $ "unknown transitions listed in " ++ source ++ "; upgrade git-annex!"
+	badsource = giveup $ "unknown transitions listed in " ++ source ++ "; upgrade git-annex!"
 
 showTransitionLine :: TransitionLine -> String
 showTransitionLine (TransitionLine ts t) = unwords [show t, show ts]
diff --git a/Remote.hs b/Remote.hs
index 10c526e1e4..bcd91b7034 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -112,7 +112,7 @@ byUUID u = headMaybe . filter matching <$> remoteList
  -}
 byName :: Maybe RemoteName -> Annex (Maybe Remote)
 byName Nothing = return Nothing
-byName (Just n) = either error Just <$> byName' n
+byName (Just n) = either giveup Just <$> byName' n
 
 {- Like byName, but the remote must have a configured UUID. -}
 byNameWithUUID :: Maybe RemoteName -> Annex (Maybe Remote)
@@ -120,7 +120,7 @@ byNameWithUUID = checkuuid <=< byName
   where
 	checkuuid Nothing = return Nothing
 	checkuuid (Just r)
-		| uuid r == NoUUID = error $
+		| uuid r == NoUUID = giveup $
 			if remoteAnnexIgnore (gitconfig r)
 				then noRemoteUUIDMsg r ++
 					" (" ++ show (remoteConfig (repo r) "ignore") ++
@@ -156,7 +156,7 @@ noRemoteUUIDMsg r = "cannot determine uuid for " ++ name r ++ " (perhaps you nee
  - and returns its UUID. Finds even repositories that are not
  - configured in .git/config. -}
 nameToUUID :: RemoteName -> Annex UUID
-nameToUUID = either error return <=< nameToUUID'
+nameToUUID = either giveup return <=< nameToUUID'
 
 nameToUUID' :: RemoteName -> Annex (Either String UUID)
 nameToUUID' "." = Right <$> getUUID -- special case for current repo
diff --git a/Remote/BitTorrent.hs b/Remote/BitTorrent.hs
index a0ccf99df2..899c57e3eb 100644
--- a/Remote/BitTorrent.hs
+++ b/Remote/BitTorrent.hs
@@ -111,7 +111,7 @@ dropKey k = do
  - implemented, it tells us nothing about the later state of the torrent.
  -}
 checkKey :: Key -> Annex Bool
-checkKey = error "cannot reliably check torrent status"
+checkKey = giveup "cannot reliably check torrent status"
 
 getBitTorrentUrls :: Key -> Annex [URLString]
 getBitTorrentUrls key = filter supported <$> getUrls key
@@ -138,7 +138,7 @@ checkTorrentUrl u = do
 	registerTorrentCleanup u
 	ifM (downloadTorrentFile u)
 		( torrentContents u
-		, error "could not download torrent file"
+		, giveup "could not download torrent file"
 		)
 
 {- To specify which file inside a multi-url torrent, the file number is
@@ -268,13 +268,13 @@ downloadTorrentContent k u dest filenum p = do
 		fs <- liftIO $ map fst <$> torrentFileSizes torrent
 		if length fs >= filenum
 			then return (fs !! (filenum - 1))
-			else error "Number of files in torrent seems to have changed."
+			else giveup "Number of files in torrent seems to have changed."
 
 checkDependencies :: Annex ()
 checkDependencies = do
 	missing <- liftIO $ filterM (not <$$> inPath) deps
 	unless (null missing) $
-		error $ "need to install additional software in order to download from bittorrent: " ++ unwords missing
+		giveup $ "need to install additional software in order to download from bittorrent: " ++ unwords missing
   where
 	deps =
 		[ "aria2c"
@@ -343,7 +343,7 @@ torrentFileSizes torrent = do
 	let mkfile = joinPath . map (scrub . decodeBS)
 	b <- B.readFile torrent
 	return $ case readTorrent b of
-		Left e -> error $ "failed to parse torrent: " ++ e
+		Left e -> giveup $ "failed to parse torrent: " ++ e
 		Right t -> case tInfo t of
 			SingleFile { tLength = l, tName = f } ->
 				[ (mkfile [f], l) ]
@@ -366,7 +366,7 @@ torrentFileSizes torrent = do
 				_ -> parsefailed (show v)
   where
 	getfield = btshowmetainfo torrent
-	parsefailed s = error $ "failed to parse btshowmetainfo output for torrent file: " ++ show s
+	parsefailed s = giveup $ "failed to parse btshowmetainfo output for torrent file: " ++ show s
 
 	-- btshowmetainfo outputs a list of "filename (size)"
 	splitsize d l = (scrub (d </> fn), sz)
@@ -379,7 +379,7 @@ torrentFileSizes torrent = do
 #endif
 	-- a malicious torrent file might try to do directory traversal
 	scrub f = if isAbsolute f || any (== "..") (splitPath f)
-		then error "found unsafe filename in torrent!"
+		then giveup "found unsafe filename in torrent!"
 		else f
 
 torrentContents :: URLString -> Annex UrlContents
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 22510859c0..332e8d5dc6 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -84,7 +84,7 @@ gen r u c gc = do
 		(simplyPrepare $ checkKey r bupr')
 		this
   where
-	buprepo = fromMaybe (error "missing buprepo") $ remoteAnnexBupRepo gc
+	buprepo = fromMaybe (giveup "missing buprepo") $ remoteAnnexBupRepo gc
 	specialcfg = (specialRemoteCfg c)
 		-- chunking would not improve bup
 		{ chunkConfig = NoChunks
@@ -95,14 +95,14 @@ bupSetup mu _ c gc = do
 	u <- maybe (liftIO genUUID) return mu
 
 	-- verify configuration is sane
-	let buprepo = fromMaybe (error "Specify buprepo=") $
+	let buprepo = fromMaybe (giveup "Specify buprepo=") $
 		M.lookup "buprepo" c
 	(c', _encsetup) <- encryptionSetup c gc
 
 	-- bup init will create the repository.
 	-- (If the repository already exists, bup init again appears safe.)
 	showAction "bup init"
-	unlessM (bup "init" buprepo []) $ error "bup init failed"
+	unlessM (bup "init" buprepo []) $ giveup "bup init failed"
 
 	storeBupUUID u buprepo
 
@@ -197,7 +197,7 @@ storeBupUUID u buprepo = do
 			showAction "storing uuid"
 			unlessM (onBupRemote r boolSystem "git"
 				[Param "config", Param "annex.uuid", Param v]) $
-					error "ssh failed"
+					giveup "ssh failed"
 		else liftIO $ do
 			r' <- Git.Config.read r
 			let olduuid = Git.Config.get "annex.uuid" "" r'
@@ -251,7 +251,7 @@ bup2GitRemote r
 	| bupLocal r = 
 		if "/" `isPrefixOf` r
 			then Git.Construct.fromAbsPath r
-			else error "please specify an absolute path"
+			else giveup "please specify an absolute path"
 	| otherwise = Git.Construct.fromUrl $ "ssh://" ++ host ++ slash dir
   where
 	bits = split ":" r
diff --git a/Remote/Ddar.hs b/Remote/Ddar.hs
index fded8d4200..dcb16f5ddc 100644
--- a/Remote/Ddar.hs
+++ b/Remote/Ddar.hs
@@ -76,7 +76,7 @@ gen r u c gc = do
 		, claimUrl = Nothing
 		, checkUrl = Nothing
 		}
-	ddarrepo = maybe (error "missing ddarrepo") (DdarRepo gc) (remoteAnnexDdarRepo gc)
+	ddarrepo = maybe (giveup "missing ddarrepo") (DdarRepo gc) (remoteAnnexDdarRepo gc)
 	specialcfg = (specialRemoteCfg c)
 		-- chunking would not improve ddar
 		{ chunkConfig = NoChunks
@@ -87,7 +87,7 @@ ddarSetup mu _ c gc = do
 	u <- maybe (liftIO genUUID) return mu
 
 	-- verify configuration is sane
-	let ddarrepo = fromMaybe (error "Specify ddarrepo=") $
+	let ddarrepo = fromMaybe (giveup "Specify ddarrepo=") $
 		M.lookup "ddarrepo" c
 	(c', _encsetup) <- encryptionSetup c gc
 
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 3b26947b63..248e5d49f7 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -75,17 +75,17 @@ gen r u c gc = do
 			, checkUrl = Nothing
 			}
   where
-	dir = fromMaybe (error "missing directory") $ remoteAnnexDirectory gc
+	dir = fromMaybe (giveup "missing directory") $ remoteAnnexDirectory gc
 
 directorySetup :: Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID)
 directorySetup mu _ c gc = do
 	u <- maybe (liftIO genUUID) return mu
 	-- verify configuration is sane
-	let dir = fromMaybe (error "Specify directory=") $
+	let dir = fromMaybe (giveup "Specify directory=") $
 		M.lookup "directory" c
 	absdir <- liftIO $ absPath dir
 	liftIO $ unlessM (doesDirectoryExist absdir) $
-		error $ "Directory does not exist: " ++ absdir
+		giveup $ "Directory does not exist: " ++ absdir
 	(c', _encsetup) <- encryptionSetup c gc
 
 	-- The directory is stored in git config, not in this remote's
@@ -216,6 +216,6 @@ checkKey d _ k = liftIO $
 		( return True
 		, ifM (doesDirectoryExist d)
 			( return False
-			, error $ "directory " ++ d ++ " is not accessible"
+			, giveup $ "directory " ++ d ++ " is not accessible"
 			)
 		)
diff --git a/Remote/External.hs b/Remote/External.hs
index 65b05fe624..0b0e1dc18b 100644
--- a/Remote/External.hs
+++ b/Remote/External.hs
@@ -107,12 +107,12 @@ gen r u c gc
 			(simplyPrepare toremove)
 			(simplyPrepare tocheckkey)
 			rmt
-	externaltype = fromMaybe (error "missing externaltype") (remoteAnnexExternalType gc)
+	externaltype = fromMaybe (giveup "missing externaltype") (remoteAnnexExternalType gc)
 
 externalSetup :: Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID)
 externalSetup mu _ c gc = do
 	u <- maybe (liftIO genUUID) return mu
-	let externaltype = fromMaybe (error "Specify externaltype=") $
+	let externaltype = fromMaybe (giveup "Specify externaltype=") $
 		M.lookup "externaltype" c
 	(c', _encsetup) <- encryptionSetup c gc
 
@@ -124,7 +124,7 @@ externalSetup mu _ c gc = do
 			external <- newExternal externaltype u c' gc
 			handleRequest external INITREMOTE Nothing $ \resp -> case resp of
 				INITREMOTE_SUCCESS -> Just noop
-				INITREMOTE_FAILURE errmsg -> Just $ error errmsg
+				INITREMOTE_FAILURE errmsg -> Just $ giveup errmsg
 				_ -> Nothing
 			withExternalState external $
 				liftIO . atomically . readTVar . externalConfig
@@ -151,8 +151,7 @@ retrieve external = fileRetriever $ \d k p ->
 			TRANSFER_SUCCESS Download k'
 				| k == k' -> Just $ return ()
 			TRANSFER_FAILURE Download k' errmsg
-				| k == k' -> Just $ do
-					error errmsg
+				| k == k' -> Just $ giveup errmsg
 			_ -> Nothing
 
 remove :: External -> Remover
@@ -168,7 +167,7 @@ remove external k = safely $
 			_ -> Nothing
 
 checkKey :: External -> CheckPresent
-checkKey external k = either error id <$> go
+checkKey external k = either giveup id <$> go
   where
 	go = handleRequest external (CHECKPRESENT k) Nothing $ \resp ->
 		case resp of
@@ -284,7 +283,7 @@ handleRequest' st external req mp responsehandler
 	handleRemoteRequest (VERSION _) =
 		sendMessage st external (ERROR "too late to send VERSION")
 
-	handleAsyncMessage (ERROR err) = error $ "external special remote error: " ++ err
+	handleAsyncMessage (ERROR err) = giveup $ "external special remote error: " ++ err
 
 	send = sendMessage st external
 
@@ -332,7 +331,7 @@ receiveMessage st external handleresponse handlerequest handleasync =
 				Nothing -> case parseMessage s :: Maybe AsyncMessage of
 					Just msg -> maybe (protocolError True s) id (handleasync msg)
 					Nothing -> protocolError False s
-	protocolError parsed s = error $ "external special remote protocol error, unexpectedly received \"" ++ s ++ "\" " ++
+	protocolError parsed s = giveup $ "external special remote protocol error, unexpectedly received \"" ++ s ++ "\" " ++
 		if parsed then "(command not allowed at this time)" else "(unable to parse command)"
 
 protocolDebug :: External -> ExternalState -> Bool -> String -> IO ()
@@ -413,14 +412,14 @@ startExternal external = do
 		environ <- propGitEnv g
 		return $ p { env = Just environ }
 
-	runerr _ = error ("Cannot run " ++ basecmd ++ " -- Make sure it's in your PATH and is executable.")
+	runerr _ = giveup ("Cannot run " ++ basecmd ++ " -- Make sure it's in your PATH and is executable.")
 
 	checkearlytermination Nothing = noop
 	checkearlytermination (Just exitcode) = ifM (inPath basecmd)
-		( error $ unwords [ "failed to run", basecmd, "(" ++ show exitcode ++ ")" ]
+		( giveup $ unwords [ "failed to run", basecmd, "(" ++ show exitcode ++ ")" ]
 		, do
 			path <- intercalate ":" <$> getSearchPath
-			error $ basecmd ++ " is not installed in PATH (" ++ path ++ ")"
+			giveup $ basecmd ++ " is not installed in PATH (" ++ path ++ ")"
 		)
 
 stopExternal :: External -> Annex ()
@@ -452,7 +451,7 @@ checkPrepared st external = do
 	v <- liftIO $ atomically $ readTVar $ externalPrepared st
 	case v of
 		Prepared -> noop
-		FailedPrepare errmsg -> error errmsg
+		FailedPrepare errmsg -> giveup errmsg
 		Unprepared ->
 			handleRequest' st external PREPARE Nothing $ \resp ->
 				case resp of
@@ -460,7 +459,7 @@ checkPrepared st external = do
 						setprepared Prepared
 					PREPARE_FAILURE errmsg -> Just $ do
 						setprepared $ FailedPrepare errmsg
-						error errmsg
+						giveup errmsg
 					_ -> Nothing
   where
 	setprepared status = liftIO $ atomically $ void $
@@ -520,8 +519,8 @@ checkurl external url =
 		CHECKURL_MULTI ((_, sz, f):[]) ->
 			Just $ return $ UrlContents sz $ Just $ mkSafeFilePath f
 		CHECKURL_MULTI l -> Just $ return $ UrlMulti $ map mkmulti l
-		CHECKURL_FAILURE errmsg -> Just $ error errmsg
-		UNSUPPORTED_REQUEST -> error "CHECKURL not implemented by external special remote"
+		CHECKURL_FAILURE errmsg -> Just $ giveup errmsg
+		UNSUPPORTED_REQUEST -> giveup "CHECKURL not implemented by external special remote"
 		_ -> Nothing
   where
 	mkmulti (u, s, f) = (u, s, mkSafeFilePath f)
@@ -530,7 +529,7 @@ retrieveUrl :: Retriever
 retrieveUrl = fileRetriever $ \f k p -> do
 	us <- getWebUrls k
 	unlessM (downloadUrl k p us f) $
-		error "failed to download content"
+		giveup "failed to download content"
 
 checkKeyUrl :: Git.Repo -> CheckPresent
 checkKeyUrl r k = do
diff --git a/Remote/GCrypt.hs b/Remote/GCrypt.hs
index a0c8ecaf7a..78ab6ed79c 100644
--- a/Remote/GCrypt.hs
+++ b/Remote/GCrypt.hs
@@ -164,16 +164,16 @@ rsyncTransport r gc
 	othertransport = return ([], loc, AccessDirect)
 
 noCrypto :: Annex a
-noCrypto = error "cannot use gcrypt remote without encryption enabled"
+noCrypto = giveup "cannot use gcrypt remote without encryption enabled"
 
 unsupportedUrl :: a
-unsupportedUrl = error "using non-ssh remote repo url with gcrypt is not supported"
+unsupportedUrl = giveup "using non-ssh remote repo url with gcrypt is not supported"
 
 gCryptSetup :: Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID)
 gCryptSetup mu _ c gc = go $ M.lookup "gitrepo" c
   where
 	remotename = fromJust (M.lookup "name" c)
-	go Nothing = error "Specify gitrepo="
+	go Nothing = giveup "Specify gitrepo="
 	go (Just gitrepo) = do
 		(c', _encsetup) <- encryptionSetup c gc
 		inRepo $ Git.Command.run 
@@ -200,7 +200,7 @@ gCryptSetup mu _ c gc = go $ M.lookup "gitrepo" c
 			]
 		g <- inRepo Git.Config.reRead
 		case Git.GCrypt.remoteRepoId g (Just remotename) of
-			Nothing -> error "unable to determine gcrypt-id of remote"
+			Nothing -> giveup "unable to determine gcrypt-id of remote"
 			Just gcryptid -> do
 				let u = genUUIDInNameSpace gCryptNameSpace gcryptid
 				if Just u == mu || isNothing mu
@@ -208,7 +208,7 @@ gCryptSetup mu _ c gc = go $ M.lookup "gitrepo" c
 						method <- setupRepo gcryptid =<< inRepo (Git.Construct.fromRemoteLocation gitrepo)
 						gitConfigSpecialRemote u c' "gcrypt" (fromAccessMethod method)
 						return (c', u)
-					else error $ "uuid mismatch; expected " ++ show mu ++ " but remote gitrepo has " ++ show u ++ " (" ++ show gcryptid ++ ")"
+					else giveup $ "uuid mismatch; expected " ++ show mu ++ " but remote gitrepo has " ++ show u ++ " (" ++ show gcryptid ++ ")"
 
 {- Sets up the gcrypt repository. The repository is either a local
  - repo, or it is accessed via rsync directly, or it is accessed over ssh
@@ -258,7 +258,7 @@ setupRepo gcryptid r
 			, Param rsyncurl
 			]
 		unless ok $
-			error "Failed to connect to remote to set it up."
+			giveup "Failed to connect to remote to set it up."
 		return AccessDirect
 
 	{-  Ask git-annex-shell to configure the repository as a gcrypt
@@ -337,7 +337,7 @@ retrieve r rsyncopts
 	| Git.repoIsSsh (repo r) = if accessShell r
 		then fileRetriever $ \f k p ->
 			unlessM (Ssh.rsyncHelper (Just p) =<< Ssh.rsyncParamsRemote False r Download k f Nothing) $
-				error "rsync failed"
+				giveup "rsync failed"
 		else fileRetriever $ Remote.Rsync.retrieve rsyncopts
 	| otherwise = unsupportedUrl
   where
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 34bdd83a16..3304e2069f 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -95,20 +95,20 @@ list autoinit = do
  -}
 gitSetup :: Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID)
 gitSetup Nothing _ c _ = do
-	let location = fromMaybe (error "Specify location=url") $
+	let location = fromMaybe (giveup "Specify location=url") $
 		Url.parseURIRelaxed =<< M.lookup "location" c
 	g <- Annex.gitRepo
 	u <- case filter (\r -> Git.location r == Git.Url location) (Git.remotes g) of
 		[r] -> getRepoUUID r
-		[] -> error "could not find existing git remote with specified location"
-		_ -> error "found multiple git remotes with specified location"
+		[] -> giveup "could not find existing git remote with specified location"
+		_ -> giveup "found multiple git remotes with specified location"
 	return (c, u)
 gitSetup (Just u) _ c _ = do
 	inRepo $ Git.Command.run
 		[ Param "remote"
 		, Param "add"
-		, Param $ fromMaybe (error "no name") (M.lookup "name" c)
-		, Param $ fromMaybe (error "no location") (M.lookup "location" c)
+		, Param $ fromMaybe (giveup "no name") (M.lookup "name" c)
+		, Param $ fromMaybe (giveup "no location") (M.lookup "location" c)
 		]
 	return (c, u)
 
@@ -202,7 +202,7 @@ tryGitConfigRead :: Bool -> Git.Repo -> Annex Git.Repo
 tryGitConfigRead autoinit r 
 	| haveconfig r = return r -- already read
 	| Git.repoIsSsh r = store $ do
-		v <- Ssh.onRemote r (pipedconfig, return (Left $ error "configlist failed")) "configlist" [] configlistfields
+		v <- Ssh.onRemote r (pipedconfig, return (Left $ giveup "configlist failed")) "configlist" [] configlistfields
 		case v of
 			Right r'
 				| haveconfig r' -> return r'
@@ -321,7 +321,7 @@ inAnnex rmt key
 		showChecking r
 		ifM (Url.withUrlOptions $ \uo -> anyM (\u -> Url.checkBoth u (keySize key) uo) (keyUrls rmt key))
 			( return True
-			, error "not found"
+			, giveup "not found"
 			)
 	checkremote = Ssh.inAnnex r key
 	checklocal = guardUsable r (cantCheck r) $
@@ -357,7 +357,7 @@ dropKey r key
 					logStatus key InfoMissing
 					Annex.Content.saveState True
 				return True
-	| Git.repoIsHttp (repo r) = error "dropping from http remote not supported"
+	| Git.repoIsHttp (repo r) = giveup "dropping from http remote not supported"
 	| otherwise = commitOnCleanup r $ Ssh.dropKey (repo r) key
 
 lockKey :: Remote -> Key -> (VerifiedCopy -> Annex r) -> Annex r
@@ -414,7 +414,7 @@ lockKey r key callback
 					failedlock
 	| otherwise = failedlock
   where
-	failedlock = error "can't lock content"
+	failedlock = giveup "can't lock content"
 
 {- Tries to copy a key's content from a remote's annex to a file. -}
 copyFromRemote :: Remote -> Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex (Bool, Verification)
@@ -444,7 +444,7 @@ copyFromRemote' r key file dest meterupdate
 	| Git.repoIsSsh (repo r) = unVerified $ feedprogressback $ \p -> do
 		Ssh.rsyncHelper (Just (combineMeterUpdate meterupdate p))
 			=<< Ssh.rsyncParamsRemote False r Download key dest file
-	| otherwise = error "copying from non-ssh, non-http remote not supported"
+	| otherwise = giveup "copying from non-ssh, non-http remote not supported"
   where
 	{- Feed local rsync's progress info back to the remote,
 	 - by forking a feeder thread that runs
@@ -547,7 +547,7 @@ copyToRemote' r key file meterupdate
 			unlocked <- isDirect <||> versionSupportsUnlockedPointers
 			Ssh.rsyncHelper (Just meterupdate)
 				=<< Ssh.rsyncParamsRemote unlocked r Upload key object file
-	| otherwise = error "copying to non-ssh repo not supported"
+	| otherwise = giveup "copying to non-ssh repo not supported"
   where
 	copylocal Nothing = return False
 	copylocal (Just (object, checksuccess)) = do
diff --git a/Remote/Glacier.hs b/Remote/Glacier.hs
index eae2dab684..77a907b97c 100644
--- a/Remote/Glacier.hs
+++ b/Remote/Glacier.hs
@@ -146,7 +146,7 @@ retrieve r k sink = go =<< glacierEnv c gc u
 		, Param $ getVault $ config r
 		, Param $ archive r k
 		]
-	go Nothing = error "cannot retrieve from glacier"
+	go Nothing = giveup "cannot retrieve from glacier"
 	go (Just e) = do
 		let cmd = (proc "glacier" (toCommand params))
 			{ env = Just e
@@ -182,7 +182,7 @@ checkKey r k = do
 	showChecking r
 	go =<< glacierEnv (config r) (gitconfig r) (uuid r)
   where
-	go Nothing = error "cannot check glacier"
+	go Nothing = giveup "cannot check glacier"
 	go (Just e) = do
 		{- glacier checkpresent outputs the archive name to stdout if
 		 - it's present. -}
@@ -190,7 +190,7 @@ checkKey r k = do
 		let probablypresent = key2file k `elem` lines s
 		if probablypresent
 			then ifM (Annex.getFlag "trustglacier")
-				( return True, error untrusted )
+				( return True, giveup untrusted )
 			else return False
 
 	params = glacierParams (config r)
@@ -222,7 +222,7 @@ glacierParams :: RemoteConfig -> [CommandParam] -> [CommandParam]
 glacierParams c params = datacenter:params
   where
 	datacenter = Param $ "--region=" ++
-		fromMaybe (error "Missing datacenter configuration")
+		fromMaybe (giveup "Missing datacenter configuration")
 			(M.lookup "datacenter" c)
 
 glacierEnv :: RemoteConfig -> RemoteGitConfig -> UUID -> Annex (Maybe [(String, String)])
@@ -239,7 +239,7 @@ glacierEnv c gc u = do
 	(uk, pk) = credPairEnvironment creds
 
 getVault :: RemoteConfig -> Vault
-getVault = fromMaybe (error "Missing vault configuration") 
+getVault = fromMaybe (giveup "Missing vault configuration") 
 	. M.lookup "vault"
 
 archive :: Remote -> Key -> Archive
@@ -249,7 +249,7 @@ archive r k = fileprefix ++ key2file k
 
 genVault :: RemoteConfig -> RemoteGitConfig -> UUID -> Annex ()
 genVault c gc u = unlessM (runGlacier c gc u params) $
-	error "Failed creating glacier vault."
+	giveup "Failed creating glacier vault."
   where
 	params = 
 		[ Param "vault"
@@ -312,7 +312,7 @@ jobList r keys = go =<< glacierEnv (config r) (gitconfig r) (uuid r)
 checkSaneGlacierCommand :: IO ()
 checkSaneGlacierCommand = 
 	whenM ((Nothing /=) <$> catchMaybeIO shouldfail) $
-		error wrongcmd
+		giveup wrongcmd
   where
 	test = proc "glacier" ["--compatibility-test-git-annex"]
 	shouldfail = withQuietOutput createProcessSuccess test
diff --git a/Remote/Helper/Chunked.hs b/Remote/Helper/Chunked.hs
index e3cf0d27b8..103dcf4ca1 100644
--- a/Remote/Helper/Chunked.hs
+++ b/Remote/Helper/Chunked.hs
@@ -59,7 +59,7 @@ getChunkConfig m =
 		Just size
 			| size == 0 -> NoChunks
 			| size > 0 -> c (fromInteger size)
-		_ -> error $ "bad configuration " ++ f ++ "=" ++ v
+		_ -> giveup $ "bad configuration " ++ f ++ "=" ++ v
 
 -- An infinite stream of chunk keys, starting from chunk 1.
 newtype ChunkKeyStream = ChunkKeyStream [Key]
diff --git a/Remote/Helper/Encryptable.hs b/Remote/Helper/Encryptable.hs
index 05c3e38a5e..45ceae0681 100644
--- a/Remote/Helper/Encryptable.hs
+++ b/Remote/Helper/Encryptable.hs
@@ -66,14 +66,14 @@ encryptionSetup c gc = do
 			encsetup $ genEncryptedCipher cmd (c, gc) key Hybrid
 		Just "pubkey" -> encsetup $ genEncryptedCipher cmd (c, gc) key PubKey
 		Just "sharedpubkey" -> encsetup $ genSharedPubKeyCipher cmd key
-		_ -> error $ "Specify " ++ intercalate " or "
+		_ -> giveup $ "Specify " ++ intercalate " or "
 			(map ("encryption=" ++)
 				["none","shared","hybrid","pubkey", "sharedpubkey"])
 			++ "."
-	key = fromMaybe (error "Specifiy keyid=...") $ M.lookup "keyid" c
+	key = fromMaybe (giveup "Specifiy keyid=...") $ M.lookup "keyid" c
 	newkeys = maybe [] (\k -> [(True,k)]) (M.lookup "keyid+" c) ++
 		maybe [] (\k -> [(False,k)]) (M.lookup "keyid-" c)
-	cannotchange = error "Cannot set encryption type of existing remotes."
+	cannotchange = giveup "Cannot set encryption type of existing remotes."
 	-- Update an existing cipher if possible.
 	updateCipher cmd v = case v of
 		SharedCipher _ | maybe True (== "shared") encryption -> return (c', EncryptionIsSetup)
diff --git a/Remote/Helper/Http.hs b/Remote/Helper/Http.hs
index f01dfd9224..ebe0f2598e 100644
--- a/Remote/Helper/Http.hs
+++ b/Remote/Helper/Http.hs
@@ -70,7 +70,7 @@ handlePopper numchunks chunksize meterupdate h sink = do
 -- meter as it goes.
 httpBodyRetriever :: FilePath -> MeterUpdate -> Response BodyReader -> IO ()
 httpBodyRetriever dest meterupdate resp
-	| responseStatus resp /= ok200 = error $ show $ responseStatus resp
+	| responseStatus resp /= ok200 = giveup $ show $ responseStatus resp
 	| otherwise = bracket (openBinaryFile dest WriteMode) hClose (go zeroBytesProcessed)
   where
 	reader = responseBody resp
diff --git a/Remote/Helper/Messages.hs b/Remote/Helper/Messages.hs
index 484ea19552..0148257760 100644
--- a/Remote/Helper/Messages.hs
+++ b/Remote/Helper/Messages.hs
@@ -29,7 +29,7 @@ showChecking :: Describable a => a -> Annex ()
 showChecking v = showAction $ "checking " ++ describe v
 
 cantCheck :: Describable a => a -> e
-cantCheck v = error $ "unable to check " ++ describe v
+cantCheck v = giveup $ "unable to check " ++ describe v
 
 showLocking :: Describable a => a -> Annex ()
 showLocking v = showAction $ "locking " ++ describe v
diff --git a/Remote/Helper/Ssh.hs b/Remote/Helper/Ssh.hs
index 4ec772296d..dff16b6568 100644
--- a/Remote/Helper/Ssh.hs
+++ b/Remote/Helper/Ssh.hs
@@ -29,7 +29,7 @@ import Config
 toRepo :: Git.Repo -> RemoteGitConfig -> [CommandParam] -> Annex [CommandParam]
 toRepo r gc sshcmd = do
 	let opts = map Param $ remoteAnnexSshOptions gc
-	let host = fromMaybe (error "bad ssh url") $ Git.Url.hostuser r
+	let host = fromMaybe (giveup "bad ssh url") $ Git.Url.hostuser r
 	params <- sshOptions (host, Git.Url.port r) gc opts
 	return $ params ++ Param host : sshcmd
 
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 7d8f7f0967..6abffe1177 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -68,12 +68,12 @@ gen r u c gc = do
 			, checkUrl = Nothing
 			}
   where
-	hooktype = fromMaybe (error "missing hooktype") $ remoteAnnexHookType gc
+	hooktype = fromMaybe (giveup "missing hooktype") $ remoteAnnexHookType gc
 
 hookSetup :: Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, UUID)
 hookSetup mu _ c gc = do
 	u <- maybe (liftIO genUUID) return mu
-	let hooktype = fromMaybe (error "Specify hooktype=") $
+	let hooktype = fromMaybe (giveup "Specify hooktype=") $
 		M.lookup "hooktype" c
 	(c', _encsetup) <- encryptionSetup c gc
 	gitConfigSpecialRemote u c' "hooktype" hooktype
@@ -129,7 +129,7 @@ store h = fileStorer $ \k src _p ->
 retrieve :: HookName -> Retriever
 retrieve h = fileRetriever $ \d k _p ->
 	unlessM (runHook h "retrieve" k (Just d) $ return True) $
-		error "failed to retrieve content"
+		giveup "failed to retrieve content"
 
 retrieveCheap :: HookName -> Key -> AssociatedFile -> FilePath -> Annex Bool
 retrieveCheap _ _ _ _ = return False
@@ -145,7 +145,7 @@ checkKey r h k = do
   where
 	action = "checkpresent"
 	findkey s = key2file k `elem` lines s
-	check Nothing = error $ action ++ " hook misconfigured"
+	check Nothing = giveup $ action ++ " hook misconfigured"
 	check (Just hook) = do
 		environ <- hookEnv action k Nothing
 		findkey <$> readProcessEnv "sh" ["-c", hook] environ
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 4695ac7a9e..22ef0b2cfb 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -53,7 +53,7 @@ gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remot
 gen r u c gc = do
 	cst <- remoteCost gc expensiveRemoteCost
 	(transport, url) <- rsyncTransport gc $
-		fromMaybe (error "missing rsyncurl") $ remoteAnnexRsyncUrl gc
+		fromMaybe (giveup "missing rsyncurl") $ remoteAnnexRsyncUrl gc
 	let o = genRsyncOpts c gc transport url
 	let islocal = rsyncUrlIsPath $ rsyncUrl o
 	return $ Just $ specialRemote' specialcfg c
@@ -127,7 +127,7 @@ rsyncTransport gc url
 					(map Param $ loginopt ++ sshopts')
 			"rsh":rshopts -> return $ map Param $ "rsh" :
 				loginopt ++ rshopts
-			rsh -> error $ "Unknown Rsync transport: "
+			rsh -> giveup $ "Unknown Rsync transport: "
 				++ unwords rsh
 	| otherwise = return ([], url)
   where
@@ -141,7 +141,7 @@ rsyncSetup :: Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig ->
 rsyncSetup mu _ c gc = do
 	u <- maybe (liftIO genUUID) return mu
 	-- verify configuration is sane
-	let url = fromMaybe (error "Specify rsyncurl=") $
+	let url = fromMaybe (giveup "Specify rsyncurl=") $
 		M.lookup "rsyncurl" c
 	(c', _encsetup) <- encryptionSetup c gc
 
@@ -188,7 +188,7 @@ store o k src meterupdate = withRsyncScratchDir $ \tmp -> do
 retrieve :: RsyncOpts -> FilePath -> Key -> MeterUpdate -> Annex ()
 retrieve o f k p = 
 	unlessM (rsyncRetrieve o k f (Just p)) $
-		error "rsync failed"
+		giveup "rsync failed"
 
 retrieveCheap :: RsyncOpts -> Key -> AssociatedFile -> FilePath -> Annex Bool
 retrieveCheap o k _af f = ifM (preseedTmp k f) ( rsyncRetrieve o k f Nothing , return False )
diff --git a/Remote/S3.hs b/Remote/S3.hs
index 97265e1481..c6f23333f1 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -136,7 +136,7 @@ s3Setup' new u mcreds c gc
 		-- Ensure user enters a valid bucket name, since
 		-- this determines the name of the archive.org item.
 		let validbucket = replace " " "-" $
-			fromMaybe (error "specify bucket=") $
+			fromMaybe (giveup "specify bucket=") $
 				getBucketName c'
 		let archiveconfig = 
 			-- IA acdepts x-amz-* as an alias for x-archive-*
@@ -252,7 +252,7 @@ retrieve r info Nothing = case getpublicurl info of
 		return False
 	Just geturl -> fileRetriever $ \f k p ->
 		unlessM (downloadUrl k p [geturl k] f) $
-			error "failed to download content"
+			giveup "failed to download content"
 
 retrieveCheap :: Key -> AssociatedFile -> FilePath -> Annex Bool
 retrieveCheap _ _ _ = return False
@@ -301,7 +301,7 @@ checkKey r info (Just h) k = do
 checkKey r info Nothing k = case getpublicurl info of
 	Nothing -> do
 		warnMissingCredPairFor "S3" (AWS.creds $ uuid r)
-		error "No S3 credentials configured"
+		giveup "No S3 credentials configured"
 	Just geturl -> do
 		showChecking r
 		withUrlOptions $ checkBoth (geturl k) (keySize k)
@@ -415,7 +415,7 @@ withS3Handle c gc u a = withS3HandleMaybe c gc u $ \mh -> case mh of
 	Just h -> a h
 	Nothing -> do
 		warnMissingCredPairFor "S3" (AWS.creds u)
-		error "No S3 credentials configured"
+		giveup "No S3 credentials configured"
 
 withS3HandleMaybe :: RemoteConfig -> RemoteGitConfig -> UUID -> (Maybe S3Handle -> Annex a) -> Annex a
 withS3HandleMaybe c gc u a = do
@@ -437,7 +437,7 @@ s3Configuration c = cfg
 	{ S3.s3Port = port
 	, S3.s3RequestStyle = case M.lookup "requeststyle" c of
 		Just "path" -> S3.PathStyle
-		Just s -> error $ "bad S3 requeststyle value: " ++ s
+		Just s -> giveup $ "bad S3 requeststyle value: " ++ s
 		Nothing -> S3.s3RequestStyle cfg
 	}
   where
@@ -455,7 +455,7 @@ s3Configuration c = cfg
 	port = let s = fromJust $ M.lookup "port" c in
 		case reads s of
 		[(p, _)] -> p
-		_ -> error $ "bad S3 port value: " ++ s
+		_ -> giveup $ "bad S3 port value: " ++ s
 	cfg = S3.s3 proto endpoint False
 
 tryS3 :: Annex a -> Annex (Either S3.S3Error a)
@@ -475,7 +475,7 @@ data S3Info = S3Info
 extractS3Info :: RemoteConfig -> Annex S3Info
 extractS3Info c = do
 	b <- maybe
-		(error "S3 bucket not configured")
+		(giveup "S3 bucket not configured")
 		(return . T.pack)
 		(getBucketName c)
 	let info = S3Info
diff --git a/Remote/Tahoe.hs b/Remote/Tahoe.hs
index 05b120d461..c29cfb438f 100644
--- a/Remote/Tahoe.hs
+++ b/Remote/Tahoe.hs
@@ -109,7 +109,7 @@ tahoeSetup mu _ c _ = do
   where
 	scsk = "shared-convergence-secret"
 	furlk = "introducer-furl"
-	missingfurl = error "Set TAHOE_FURL to the introducer furl to use."
+	missingfurl = giveup "Set TAHOE_FURL to the introducer furl to use."
 
 store :: UUID -> TahoeHandle -> Key -> AssociatedFile -> MeterUpdate -> Annex Bool
 store u hdl k _f _p = sendAnnex k noop $ \src ->
@@ -137,7 +137,7 @@ checkKey u hdl k = go =<< getCapability u k
 			[ Param "--raw"
 			, Param cap
 			]
-		either error return v
+		either giveup return v
 
 defaultTahoeConfigDir :: UUID -> IO TahoeConfigDir
 defaultTahoeConfigDir u = do
@@ -147,7 +147,7 @@ defaultTahoeConfigDir u = do
 tahoeConfigure :: TahoeConfigDir -> IntroducerFurl -> Maybe SharedConvergenceSecret -> IO SharedConvergenceSecret
 tahoeConfigure configdir furl mscs = do
 	unlessM (createClient configdir furl) $
-		error "tahoe create-client failed"
+		giveup "tahoe create-client failed"
 	maybe noop (writeSharedConvergenceSecret configdir) mscs
 	startTahoeDaemon configdir
 	getSharedConvergenceSecret configdir
@@ -173,7 +173,7 @@ getSharedConvergenceSecret configdir = go (60 :: Int)
   where
 	f = convergenceFile configdir
 	go n
-		| n == 0 = error $ "tahoe did not write " ++ f ++ " after 1 minute. Perhaps the daemon failed to start?"
+		| n == 0 = giveup $ "tahoe did not write " ++ f ++ " after 1 minute. Perhaps the daemon failed to start?"
 		| otherwise = do
 			v <- catchMaybeIO (readFile f)
 			case v of
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 033057dd8d..be2f265e08 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -100,7 +100,7 @@ checkKey key = do
 	us <- getWebUrls key
 	if null us
 		then return False
-		else either error return =<< checkKey' key us
+		else either giveup return =<< checkKey' key us
 checkKey' :: Key -> [URLString] -> Annex (Either String Bool)
 checkKey' key us = firsthit us (Right False) $ \u -> do
 	let (u', downloader) = getDownloader u
diff --git a/Remote/WebDAV.hs b/Remote/WebDAV.hs
index 3de8b357e6..19dbaa8af5 100644
--- a/Remote/WebDAV.hs
+++ b/Remote/WebDAV.hs
@@ -85,7 +85,7 @@ webdavSetup :: Maybe UUID -> Maybe CredPair -> RemoteConfig -> RemoteGitConfig -
 webdavSetup mu mcreds c gc = do
 	u <- maybe (liftIO genUUID) return mu
 	url <- case M.lookup "url" c of
-		Nothing -> error "Specify url="
+		Nothing -> giveup "Specify url="
 		Just url -> return url
 	(c', encsetup) <- encryptionSetup c gc
 	creds <- maybe (getCreds c' gc u) (return . Just) mcreds
@@ -122,7 +122,7 @@ retrieveCheap :: Key -> AssociatedFile -> FilePath -> Annex Bool
 retrieveCheap _ _ _ = return False
 
 retrieve :: ChunkConfig -> Maybe DavHandle -> Retriever
-retrieve _ Nothing = error "unable to connect"
+retrieve _ Nothing = giveup "unable to connect"
 retrieve (LegacyChunks _) (Just dav) = retrieveLegacyChunked dav
 retrieve _ (Just dav) = fileRetriever $ \d k p -> liftIO $
 	goDAV dav $
@@ -147,7 +147,7 @@ remove (Just dav) k = liftIO $ do
 					_ -> return False
 
 checkKey :: Remote -> ChunkConfig -> Maybe DavHandle -> CheckPresent
-checkKey r _ Nothing _ = error $ name r ++ " not configured"
+checkKey r _ Nothing _ = giveup $ name r ++ " not configured"
 checkKey r chunkconfig (Just dav) k = do
 	showChecking r
 	case chunkconfig of
@@ -155,7 +155,7 @@ checkKey r chunkconfig (Just dav) k = do
 		_ -> do
 			v <- liftIO $ goDAV dav $
 				existsDAV (keyLocation k)
-			either error return v
+			either giveup return v
 
 configUrl :: Remote -> Maybe URLString
 configUrl r = fixup <$> M.lookup "url" (config r)
diff --git a/Upgrade.hs b/Upgrade.hs
index 20ed7a4022..c6552f89c0 100644
--- a/Upgrade.hs
+++ b/Upgrade.hs
@@ -21,7 +21,7 @@ import qualified Upgrade.V4
 import qualified Upgrade.V5
 
 checkUpgrade :: Version -> Annex ()
-checkUpgrade = maybe noop error <=< needsUpgrade
+checkUpgrade = maybe noop giveup <=< needsUpgrade
 
 needsUpgrade :: Version -> Annex (Maybe String)
 needsUpgrade v
@@ -49,8 +49,8 @@ upgrade automatic destversion = do
 	go (Just "0") = Upgrade.V0.upgrade
 	go (Just "1") = Upgrade.V1.upgrade
 #else
-	go (Just "0") = error "upgrade from v0 on Windows not supported"
-	go (Just "1") = error "upgrade from v1 on Windows not supported"
+	go (Just "0") = giveup "upgrade from v0 on Windows not supported"
+	go (Just "1") = giveup "upgrade from v1 on Windows not supported"
 #endif
 	go (Just "2") = Upgrade.V2.upgrade
 	go (Just "3") = Upgrade.V3.upgrade automatic
diff --git a/Utility/Daemon.hs b/Utility/Daemon.hs
index 3cc2eb2613..5c0ea41696 100644
--- a/Utility/Daemon.hs
+++ b/Utility/Daemon.hs
@@ -111,7 +111,7 @@ lockPidFile pidfile = do
 #endif
 
 alreadyRunning :: IO ()
-alreadyRunning = error "Daemon is already running."
+alreadyRunning = giveup "Daemon is already running."
 
 {- Checks if the daemon is running, by checking that the pid file
  - is locked by the same process that is listed in the pid file.
@@ -135,7 +135,7 @@ checkDaemon pidfile = bracket setup cleanup go
 	check _ Nothing = Nothing
 	check (Just (pid, _)) (Just pid')
 		| pid == pid' = Just pid
-		| otherwise = error $
+		| otherwise = giveup $
 			"stale pid in " ++ pidfile ++ 
 			" (got " ++ show pid' ++ 
 			"; expected " ++ show pid ++ " )"
diff --git a/Utility/DirWatcher/FSEvents.hs b/Utility/DirWatcher/FSEvents.hs
index a07139c44b..d7472d490a 100644
--- a/Utility/DirWatcher/FSEvents.hs
+++ b/Utility/DirWatcher/FSEvents.hs
@@ -17,7 +17,7 @@ import Data.Bits ((.&.))
 watchDir :: FilePath -> (FilePath -> Bool) -> Bool -> WatchHooks -> IO EventStream
 watchDir dir ignored scanevents hooks = do
 	unlessM fileLevelEventsSupported $
-		error "Need at least OSX 10.7.0 for file-level FSEvents"
+		giveup "Need at least OSX 10.7.0 for file-level FSEvents"
 	scan dir
 	eventStreamCreate [dir] 1.0 True True True dispatch
   where
diff --git a/Utility/DirWatcher/INotify.hs b/Utility/DirWatcher/INotify.hs
index 4d11b95a8d..1890b8af5e 100644
--- a/Utility/DirWatcher/INotify.hs
+++ b/Utility/DirWatcher/INotify.hs
@@ -152,7 +152,7 @@ watchDir i dir ignored scanevents hooks
 		-- disk full error.
 		| isFullError e =
 			case errHook hooks of
-				Nothing -> error $ "failed to add inotify watch on directory " ++ dir ++ " (" ++ show e ++ ")"
+				Nothing -> giveup $ "failed to add inotify watch on directory " ++ dir ++ " (" ++ show e ++ ")"
 				Just hook -> tooManyWatches hook dir
 		-- The directory could have been deleted.
 		| isDoesNotExistError e = return ()
diff --git a/Utility/Exception.hs b/Utility/Exception.hs
index 0ffc7103fa..5cd8fd1997 100644
--- a/Utility/Exception.hs
+++ b/Utility/Exception.hs
@@ -1,6 +1,6 @@
 {- Simple IO exception handling (and some more)
  -
- - Copyright 2011-2015 Joey Hess <id@joeyh.name>
+ - Copyright 2011-2016 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}
@@ -10,6 +10,7 @@
 
 module Utility.Exception (
 	module X,
+	giveup,
 	catchBoolIO,
 	catchMaybeIO,
 	catchDefaultIO,
@@ -40,6 +41,17 @@ import GHC.IO.Exception (IOErrorType(..))
 
 import Utility.Data
 
+{- Like error, this throws an exception. Unlike error, if this exception
+ - is not caught, it won't generate a backtrace. So use this for situations
+ - where there's a problem that the user is excpected to see in some
+ - circumstances. -}
+giveup :: [Char] -> a
+#if MIN_VERSION_base(4,9,0)
+giveup = errorWithoutStackTrace
+#else
+giveup = error
+#endif
+
 {- Catches IO errors and returns a Bool -}
 catchBoolIO :: MonadCatch m => m Bool -> m Bool
 catchBoolIO = catchDefaultIO False
diff --git a/Utility/Glob.hs b/Utility/Glob.hs
index 98ffe751ba..119ea48347 100644
--- a/Utility/Glob.hs
+++ b/Utility/Glob.hs
@@ -12,6 +12,8 @@ module Utility.Glob (
 	matchGlob
 ) where
 
+import Utility.Exception
+
 import System.Path.WildMatch
 
 import "regex-tdfa" Text.Regex.TDFA
@@ -26,7 +28,7 @@ compileGlob :: String -> GlobCase -> Glob
 compileGlob glob globcase = Glob $
 	case compile (defaultCompOpt {caseSensitive = casesentitive}) defaultExecOpt regex of
 		Right r -> r
-		Left _ -> error $ "failed to compile regex: " ++ regex
+		Left _ -> giveup $ "failed to compile regex: " ++ regex
   where
 	regex = '^':wildToRegex glob
 	casesentitive = case globcase of
diff --git a/Utility/Gpg.hs b/Utility/Gpg.hs
index 21171b6fb0..1185152220 100644
--- a/Utility/Gpg.hs
+++ b/Utility/Gpg.hs
@@ -253,7 +253,7 @@ genRandom cmd highQuality size = checksize <$> readStrict cmd params
 			then s
 			else shortread len
 
-	shortread got = error $ unwords
+	shortread got = giveup $ unwords
 		[ "Not enough bytes returned from gpg", show params
 		, "(got", show got, "; expected", show expectedlength, ")"
 		]
diff --git a/Utility/LockFile/PidLock.hs b/Utility/LockFile/PidLock.hs
index 6a3e86a3f5..bc8ddfe6bb 100644
--- a/Utility/LockFile/PidLock.hs
+++ b/Utility/LockFile/PidLock.hs
@@ -210,7 +210,7 @@ waitLock (Seconds timeout) lockfile = go timeout
 			=<< tryLock lockfile
 		| otherwise = do
 			hPutStrLn stderr $ show timeout ++ " second timeout exceeded while waiting for pid lock file " ++ lockfile
-			error $ "Gave up waiting for possibly stale pid lock file " ++ lockfile
+			giveup $ "Gave up waiting for possibly stale pid lock file " ++ lockfile
 
 dropLock :: LockHandle -> IO ()
 dropLock (LockHandle lockfile _ sidelock) = do
diff --git a/Utility/Quvi.hs b/Utility/Quvi.hs
index 09f74968b0..417ab7041c 100644
--- a/Utility/Quvi.hs
+++ b/Utility/Quvi.hs
@@ -79,8 +79,8 @@ forceQuery :: Query (Maybe Page)
 forceQuery v ps url = query' v ps url `catchNonAsync` onerr
   where
 	onerr e = ifM (inPath "quvi")
-		( error ("quvi failed: " ++ show e)
-		, error "quvi is not installed"
+		( giveup ("quvi failed: " ++ show e)
+		, giveup "quvi is not installed"
 		)
 
 {- Returns Nothing if the page is not a video page, or quvi is not
diff --git a/Utility/UserInfo.hs b/Utility/UserInfo.hs
index ec0b0d0b2e..dd66c331e6 100644
--- a/Utility/UserInfo.hs
+++ b/Utility/UserInfo.hs
@@ -16,6 +16,7 @@ module Utility.UserInfo (
 
 import Utility.Env
 import Utility.Data
+import Utility.Exception
 
 import System.PosixCompat
 import Control.Applicative
@@ -25,7 +26,7 @@ import Prelude
  -
  - getpwent will fail on LDAP or NIS, so use HOME if set. -}
 myHomeDir :: IO FilePath
-myHomeDir = either error return =<< myVal env homeDirectory
+myHomeDir = either giveup return =<< myVal env homeDirectory
   where
 #ifndef mingw32_HOST_OS
 	env = ["HOME"]

From d2b49090b81b69426a2cde5313ca7702b9a7ed6e Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 15 Nov 2016 21:52:10 -0400
Subject: [PATCH 025/367] remove debug print

---
 Build/LinuxMkLibs.hs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Build/LinuxMkLibs.hs b/Build/LinuxMkLibs.hs
index d7512bfe0f..ba40206fdf 100644
--- a/Build/LinuxMkLibs.hs
+++ b/Build/LinuxMkLibs.hs
@@ -70,7 +70,6 @@ installLinkerShim top linker exe = do
 			-- Assume that for a symlink, the destination
 			-- will also be shimmed.
 			let sl' = ".." </> takeFileName sl </> takeFileName sl
-			print (sl', exedest)
 			createSymbolicLink sl' exedest
 		, renameFile exe exedest
 		)

From 2493c2c5a43b5ea869be3fe982f767d8b5e97703 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 15 Nov 2016 22:01:55 -0400
Subject: [PATCH 026/367] allow Utility.Exception to still be used when not
 building with cabal

---
 Utility/Exception.hs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/Utility/Exception.hs b/Utility/Exception.hs
index 5cd8fd1997..67c2e85d81 100644
--- a/Utility/Exception.hs
+++ b/Utility/Exception.hs
@@ -46,11 +46,15 @@ import Utility.Data
  - where there's a problem that the user is excpected to see in some
  - circumstances. -}
 giveup :: [Char] -> a
+#ifdef MIN_VERSION_base
 #if MIN_VERSION_base(4,9,0)
 giveup = errorWithoutStackTrace
 #else
 giveup = error
 #endif
+#else
+giveup = error
+#endif
 
 {- Catches IO errors and returns a Bool -}
 catchBoolIO :: MonadCatch m => m Bool -> m Bool

From 2542fb58ed23ddfa4627ac49c0b491c69dac1c6c Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 16 Nov 2016 00:28:10 -0400
Subject: [PATCH 027/367] fix giveup shadowing

---
 Remote/Helper/Chunked.hs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/Remote/Helper/Chunked.hs b/Remote/Helper/Chunked.hs
index 103dcf4ca1..f3c69c38dd 100644
--- a/Remote/Helper/Chunked.hs
+++ b/Remote/Helper/Chunked.hs
@@ -250,9 +250,9 @@ retrieveChunks retriever u chunkconfig encryptor basek dest basep sink
 		let ls' = maybe ls (setupResume ls) currsize
 		if any null ls'
 			then return True -- dest is already complete
-			else firstavail currsize ls' `catchNonAsync` giveup
+			else firstavail currsize ls' `catchNonAsync` unable
 
-	giveup e = do
+	unable e = do
 		warning (show e)
 		return False
 
@@ -273,10 +273,10 @@ retrieveChunks retriever u chunkconfig encryptor basek dest basep sink
 						let sz = toBytesProcessed $
 							fromMaybe 0 $ keyChunkSize k
 						getrest p h sz sz ks
-							`catchNonAsync` giveup
+							`catchNonAsync` unable
 			case v of
 				Left e
-					| null ls -> giveup e
+					| null ls -> unable e
 					| otherwise -> firstavail currsize ls
 				Right r -> return r
 
@@ -286,7 +286,7 @@ retrieveChunks retriever u chunkconfig encryptor basek dest basep sink
 		liftIO $ p' zeroBytesProcessed
 		ifM (retriever (encryptor k) p' $ tosink (Just h) p')
 			( getrest p h sz (addBytesProcessed bytesprocessed sz) ks
-			, giveup "chunk retrieval failed"
+			, unable "chunk retrieval failed"
 			)
 
 	getunchunked = retriever (encryptor basek) basep $ tosink Nothing basep

From 4276e713be6019b7f6caf86da4e8228ee93c9353 Mon Sep 17 00:00:00 2001
From: andrew <andrew@web>
Date: Wed, 16 Nov 2016 15:37:01 +0000
Subject: [PATCH 028/367] Added a comment: how to investigate

---
 ...mment_2_09c62e4abf4ccc0d2e030ef5e1bcdf71._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_2_09c62e4abf4ccc0d2e030ef5e1bcdf71._comment

diff --git a/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_2_09c62e4abf4ccc0d2e030ef5e1bcdf71._comment b/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_2_09c62e4abf4ccc0d2e030ef5e1bcdf71._comment
new file mode 100644
index 0000000000..fcca1b28e8
--- /dev/null
+++ b/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_2_09c62e4abf4ccc0d2e030ef5e1bcdf71._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="how to investigate"
+ date="2016-11-16T15:37:01Z"
+ content="""
+Any thoughts? I am unsure how to investigate where this problem is. I assume these files are in my git repo or git-annex objects but I can't seem to find them using any search commands.
+
+Thanks,
+
+Andrew
+"""]]

From 887e1679370ebcf7e117c6b84628eaf13b9d2dea Mon Sep 17 00:00:00 2001
From: andrew <andrew@web>
Date: Wed, 16 Nov 2016 15:55:44 +0000
Subject: [PATCH 029/367]

---
 ...7__gpg-agent__39___for_testing_failed.mdwn | 30 +++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 doc/bugs/DBG__58___running___96____47__Users__47__joey__47__homebrew__47__opt__47__gpg-agent__47__bin__47__gpg-agent__39___for_testing_failed.mdwn

diff --git a/doc/bugs/DBG__58___running___96____47__Users__47__joey__47__homebrew__47__opt__47__gpg-agent__47__bin__47__gpg-agent__39___for_testing_failed.mdwn b/doc/bugs/DBG__58___running___96____47__Users__47__joey__47__homebrew__47__opt__47__gpg-agent__47__bin__47__gpg-agent__39___for_testing_failed.mdwn
new file mode 100644
index 0000000000..a3b85bdf24
--- /dev/null
+++ b/doc/bugs/DBG__58___running___96____47__Users__47__joey__47__homebrew__47__opt__47__gpg-agent__47__bin__47__gpg-agent__39___for_testing_failed.mdwn
@@ -0,0 +1,30 @@
+### Please describe the problem.
+Issue uploading to S3 remote (Dreamhost)
+
+### What steps will reproduce the problem?
+git-annex copy massart/f16_Web1/screencaptures/IMG_5159.MOV --to=cloud
+on my repo
+
+### What version of git-annex are you using? On what operating system?
+6.20161031-g0a58e94
+OS-X 10.11.6
+
+### Please provide any additional information below.
+I am using a different WIFI I haven't used before. Maybe it is blocking something…
+
+[[!format sh """
+git-annex copy massart/f16_Web1/screencaptures/IMG_5159.MOV --to=cloud
+copy massart/f16_Web1/screencaptures/IMG_5159.MOV (checking cloud...) (to cloud...) 
+17%           0.0 B/s 0sgpg: error running `/Users/joey/homebrew/opt/gpg-agent/bin/gpg-agent': probably not installed
+gpg: DBG: running `/Users/joey/homebrew/opt/gpg-agent/bin/gpg-agent' for testing failed: Configuration error
+gpg: can't connect to the agent: IPC connect call failed
+gpg: problem with the agent: No agent running
+35%       1021.8KB/s 30s
+  user error (gpg ["--quiet","--trust-model","always","--batch","--passphrase-fd","26","--symmetric","--force-mdc","--no-textmode"] exited 2)
+failed
+git-annex: copy: 1 failed
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+Yes.
+

From 681a4e5a581e73614901cbf0e32b33097c289886 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 16 Nov 2016 14:33:33 -0400
Subject: [PATCH 030/367] comment

---
 ...ment_1_67e186265ae21f2cd8451750152f2a6d._comment | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/forum/vanilla_git_repo_as_special_remote__63__/comment_1_67e186265ae21f2cd8451750152f2a6d._comment

diff --git a/doc/forum/vanilla_git_repo_as_special_remote__63__/comment_1_67e186265ae21f2cd8451750152f2a6d._comment b/doc/forum/vanilla_git_repo_as_special_remote__63__/comment_1_67e186265ae21f2cd8451750152f2a6d._comment
new file mode 100644
index 0000000000..65397495a2
--- /dev/null
+++ b/doc/forum/vanilla_git_repo_as_special_remote__63__/comment_1_67e186265ae21f2cd8451750152f2a6d._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-11-16T18:30:12Z"
+ content="""
+You can use bup as a special remote, which will store the content in a git
+repository. But, not in a form that git diff can be used with.
+
+[[git-annex-diffdriver]] can be used to make `git diff` work on annexed
+files. For example:
+
+	export GIT_EXTERNAL_DIFF="git-annex diffdriver -- diff -u --"
+"""]]

From 6d1039822e2b478719970d9788399e45563838f9 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 16 Nov 2016 14:35:22 -0400
Subject: [PATCH 031/367] already fixed

---
 ...ingle_space_in_file_name_causes_git_annex_add_to_fail.mdwn | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/bugs/Single_space_in_file_name_causes_git_annex_add_to_fail.mdwn b/doc/bugs/Single_space_in_file_name_causes_git_annex_add_to_fail.mdwn
index 14f772c5fd..0965621994 100644
--- a/doc/bugs/Single_space_in_file_name_causes_git_annex_add_to_fail.mdwn
+++ b/doc/bugs/Single_space_in_file_name_causes_git_annex_add_to_fail.mdwn
@@ -51,3 +51,7 @@ Not sure whether this means that this bug is actually fixed or whether it's an a
 ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
 
 Just starting out with it as a way of archiving some of my media seems to work great apart from this. Thanks a bunch!
+
+> This bug was already fixed in git-annex 6.20161031. I told the Debian
+> maintainer about the bug fix at the time; package has not been updated
+> yet. [[done]] on git-annex side. --[[Joey]] 

From d7b8be75a4bd5d857bd64790ca4fcb0b59710792 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 16 Nov 2016 14:40:28 -0400
Subject: [PATCH 032/367] comment

---
 ..._9392346203c561b88f30fa2ce7540b76._comment | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 doc/bugs/When_stopping___96__git_annex_get__96___files_left_broken/comment_1_9392346203c561b88f30fa2ce7540b76._comment

diff --git a/doc/bugs/When_stopping___96__git_annex_get__96___files_left_broken/comment_1_9392346203c561b88f30fa2ce7540b76._comment b/doc/bugs/When_stopping___96__git_annex_get__96___files_left_broken/comment_1_9392346203c561b88f30fa2ce7540b76._comment
new file mode 100644
index 0000000000..4f90ddfa69
--- /dev/null
+++ b/doc/bugs/When_stopping___96__git_annex_get__96___files_left_broken/comment_1_9392346203c561b88f30fa2ce7540b76._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-11-16T18:36:34Z"
+ content="""
+Thing is, git-annex get does not update the file in place. Only once the
+entire file is downloaded, and its content is verified correct is it moved
+into a place where you can access it.
+
+So, it seems much more likely to me that the content of the file, as
+originally added to git-annex, was bad, and the it had just finished
+verifying the content and moving it into place when you interruped the
+command.
+
+Please check with `git annex fsck` on the file and see if it determines
+it has the content git-annex expects it to have.
+
+However, I notice you're using a v6 repository. Is the file an unlocked
+file? It's possible that in that specific case there could be a bug.
+I've interrupted `git annex get` on a nearly daily basis for years, but
+v6 is still experimental and not as well tested.
+"""]]

From 1b71d920e4200c73da4c1c730a7acbcdb2e62fc2 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 16 Nov 2016 14:43:33 -0400
Subject: [PATCH 033/367] comment

---
 ..._70480ffd417788f18cd75a9b625ecf3b._comment | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 doc/bugs/Adding_zip_or_7z_or_tar_archive_builds_for_windows/comment_1_70480ffd417788f18cd75a9b625ecf3b._comment

diff --git a/doc/bugs/Adding_zip_or_7z_or_tar_archive_builds_for_windows/comment_1_70480ffd417788f18cd75a9b625ecf3b._comment b/doc/bugs/Adding_zip_or_7z_or_tar_archive_builds_for_windows/comment_1_70480ffd417788f18cd75a9b625ecf3b._comment
new file mode 100644
index 0000000000..3bd7381e16
--- /dev/null
+++ b/doc/bugs/Adding_zip_or_7z_or_tar_archive_builds_for_windows/comment_1_70480ffd417788f18cd75a9b625ecf3b._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-11-16T18:41:25Z"
+ content="""
+It would be helpful to have more details, such as an example of software
+distributed for windows that way, or documentation of how such an archive
+is used on windows.
+
+The git-annex Windows installer is a exe file that uses the NullSoft
+installation system. As far as I know that's pretty common in the Windows
+world.
+
+I don't see any point in zipping up the single exe. It would be possible to
+make a zip containing all the files that instlling the exe installs. But,
+the installation process has to integrate git-annex with git, it installs
+menu items, etc. A zip file would not be able to handle that integration.
+So its use seems limited to me.
+"""]]

From b868e931a63076a6a35a601654e61d4d35005dea Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 16 Nov 2016 14:43:45 -0400
Subject: [PATCH 034/367] moreinfo

---
 .../Adding_zip_or_7z_or_tar_archive_builds_for_windows.mdwn     | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/Adding_zip_or_7z_or_tar_archive_builds_for_windows.mdwn b/doc/bugs/Adding_zip_or_7z_or_tar_archive_builds_for_windows.mdwn
index 6cca0082cb..ae1f7c5229 100644
--- a/doc/bugs/Adding_zip_or_7z_or_tar_archive_builds_for_windows.mdwn
+++ b/doc/bugs/Adding_zip_or_7z_or_tar_archive_builds_for_windows.mdwn
@@ -1,3 +1,5 @@
 Having a windows build of Git-Annex in an archived format would be very usefull for automation, and deploy.
 Could it be possible to add this to the buildserver of gitannex?
 
+[[!tag moreinfo]]
+

From 2f192583ba9666d3ce2e132e396d96c0a6bb92c5 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 16 Nov 2016 14:49:47 -0400
Subject: [PATCH 035/367] comment

---
 ...comment_1_e308245bf81a536db6f9a2b743d912bf._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_1_e308245bf81a536db6f9a2b743d912bf._comment

diff --git a/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_1_e308245bf81a536db6f9a2b743d912bf._comment b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_1_e308245bf81a536db6f9a2b743d912bf._comment
new file mode 100644
index 0000000000..a5d988faeb
--- /dev/null
+++ b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_1_e308245bf81a536db6f9a2b743d912bf._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-11-16T18:49:09Z"
+ content="""
+I'm not able to reproduce the problem with your test case and git-annex
+version 6.20161012.
+
+Can you still reproduce it after upgrading?
+"""]]

From bd4cf0f663e7a081406b4d7b2a4e2a4a289d6811 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 16 Nov 2016 15:01:49 -0400
Subject: [PATCH 036/367] comment

---
 ..._8f694afa77f5a835c826d29d46d44615._comment | 30 +++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_2_8f694afa77f5a835c826d29d46d44615._comment

diff --git a/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_2_8f694afa77f5a835c826d29d46d44615._comment b/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_2_8f694afa77f5a835c826d29d46d44615._comment
new file mode 100644
index 0000000000..ccaeeb4093
--- /dev/null
+++ b/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_2_8f694afa77f5a835c826d29d46d44615._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-11-16T18:52:07Z"
+ content="""
+It would be helpful if you said what version of git-annex you are using.
+
+And, is your git-annex repository using the new experimental v6 format? One
+user reported a similar error message with a v6 git-annex repository. See
+[[bugs/assistant_crashes_in_TransferScanner]]
+
+Or might your repository be using direct mode?
+
+So, please paste in `git annex version` and `git annex info` output.
+
+It kind of looks like it's having difficulty determining where the top of
+the git repository is, or constructing a relative path to the git
+repository.
+
+Are there any symlinks in the path to /Users/andrew/notes ? Eg, is /Users
+a symlink, or /Users/andrew a symlink, or //Users/andrew/notes itself
+symlinked to elsewhere?
+
+Does only `git annex sync --content` fail? What if you run, eg
+`git annex copy --auto --to cloud` and `git annex get --auto --from cloud`,
+does that fail similarly, or does it succeed?
+
+You say it's only failing for some files. Do the filenames that it's
+failing on contain any non-ascii characters?
+"""]]

From b4b970d5f294f6b8b444e97268eeeee6bfc6b3bd Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 16 Nov 2016 15:12:46 -0400
Subject: [PATCH 037/367] moreinfo needed

---
 ..._2744e42db662486b46e203a72c3e56c7._comment | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 doc/bugs/addurl_pathdepth_description_misleading/comment_3_2744e42db662486b46e203a72c3e56c7._comment

diff --git a/doc/bugs/addurl_pathdepth_description_misleading/comment_3_2744e42db662486b46e203a72c3e56c7._comment b/doc/bugs/addurl_pathdepth_description_misleading/comment_3_2744e42db662486b46e203a72c3e56c7._comment
new file mode 100644
index 0000000000..45be251864
--- /dev/null
+++ b/doc/bugs/addurl_pathdepth_description_misleading/comment_3_2744e42db662486b46e203a72c3e56c7._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-11-16T19:04:37Z"
+ content="""
+Seems to work as described here:
+
+	joey@darkstar:~/tmp/r>rm localhost__joey_blog_index.html 
+	joey@darkstar:~/tmp/r>git annex addurl --pathdepth 2 http://localhost/~joey/blog/index.html
+	addurl blog/index.html (downloading http://localhost/~joey/blog/index.html ...) 
+	/home/joey/tmp/r/ 100%[===========>]  40.70K  --.-KB/s    in 0s      
+	ok
+	(recording state in git...)
+	joey@darkstar:~/tmp/r>ls
+	blog/
+	joey@darkstar:~/tmp/r>ls blog
+	index.html
+
+It would probably help if you can provide a test case where it does not work
+as described.
+"""]]

From 2577f1c0a2ef91a455815ca4e812418762d83b9e Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 16 Nov 2016 15:32:49 -0400
Subject: [PATCH 038/367] fsck --all --from was checking the content of files
 in the local repository, rather than on the special remote.

Straight up forgot to handle this case!

This commit was sponsored by Fernando Jimenez on Patreon.
---
 CHANGELOG                                     |  2 ++
 Command/Fsck.hs                               | 22 ++++++++++---------
 ..._doesn__39__t_work_for_special_remote.mdwn |  2 ++
 3 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index 71ef1c1002..6dd4edb3ec 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -11,6 +11,8 @@ git-annex (6.20161112) UNRELEASED; urgency=medium
     direct mode repository work the same as it works in indirect mode.
   * Avoid backtraces on expected failures when built with ghc 8;
     only use backtraces for unexpected errors.
+  * fsck --all --from was checking the existence and content of files
+    in the local repository, rather than on the special remote. Oops.
 
  -- Joey Hess <id@joeyh.name>  Tue, 15 Nov 2016 11:15:27 -0400
 
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 9383c07f27..96ffd35da5 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -89,7 +89,7 @@ seek o = allowConcurrentOutput $ do
 	checkDeadRepo u
 	i <- prepIncremental u (incrementalOpt o)
 	withKeyOptions (keyOptions o) False
-		(\k ai -> startKey i k ai =<< getNumCopies)
+		(\k ai -> startKey from i k ai =<< getNumCopies)
 		(withFilesInGit $ whenAnnexed $ start from i)
 		(fsckFiles o)
 	cleanupIncremental i
@@ -109,7 +109,7 @@ start from inc file key = do
 			numcopies <- getFileNumCopies file
 			case from of
 				Nothing -> go $ perform key file backend numcopies
-				Just r -> go $ performRemote key file backend numcopies r
+				Just r -> go $ performRemote key (Just file) backend numcopies r
   where
 	go = runFsck inc (mkActionItem (Just file)) key
 
@@ -129,8 +129,8 @@ perform key file backend numcopies = do
 
 {- To fsck a remote, the content is retrieved to a tmp file,
  - and checked locally. -}
-performRemote :: Key -> FilePath -> Backend -> NumCopies -> Remote -> Annex Bool
-performRemote key file backend numcopies remote =
+performRemote :: Key -> AssociatedFile -> Backend -> NumCopies -> Remote -> Annex Bool
+performRemote key afile backend numcopies remote =
 	dispatch =<< Remote.hasKey remote key
   where
 	dispatch (Left err) = do
@@ -147,10 +147,10 @@ performRemote key file backend numcopies remote =
 				return False
 	dispatch (Right False) = go False Nothing
 	go present localcopy = check
-		[ verifyLocationLogRemote key file remote present
+		[ verifyLocationLogRemote key (maybe (key2file key) id afile) remote present
 		, checkKeySizeRemote key remote localcopy
 		, checkBackendRemote backend key remote localcopy
-		, checkKeyNumCopies key (Just file) numcopies
+		, checkKeyNumCopies key afile numcopies
 		]
 	withtmp a = do
 		pid <- liftIO getPID
@@ -161,7 +161,7 @@ performRemote key file backend numcopies remote =
 		cleanup
 		cleanup `after` a tmp
 	getfile tmp = ifM (checkDiskSpace (Just (takeDirectory tmp)) key 0 True)
-		( ifM (Remote.retrieveKeyFileCheap remote key (Just file) tmp)
+		( ifM (Remote.retrieveKeyFileCheap remote key afile tmp)
 			( return (Just True)
 			, ifM (Annex.getState Annex.fast)
 				( return Nothing
@@ -173,12 +173,14 @@ performRemote key file backend numcopies remote =
 		)
 	dummymeter _ = noop
 
-startKey :: Incremental -> Key -> ActionItem -> NumCopies -> CommandStart
-startKey inc key ai numcopies =
+startKey :: Maybe Remote -> Incremental -> Key -> ActionItem -> NumCopies -> CommandStart
+startKey from inc key ai numcopies =
 	case Backend.maybeLookupBackendName (keyBackendName key) of
 		Nothing -> stop
 		Just backend -> runFsck inc ai key $
-			performKey key backend numcopies
+			case from of
+				Nothing -> performKey key backend numcopies
+				Just r -> performRemote key Nothing backend numcopies r
 
 performKey :: Key -> Backend -> NumCopies -> Annex Bool
 performKey key backend numcopies = do
diff --git a/doc/bugs/git-annex-fsck___34__-all__34___flag_doesn__39__t_work_for_special_remote.mdwn b/doc/bugs/git-annex-fsck___34__-all__34___flag_doesn__39__t_work_for_special_remote.mdwn
index b9895056fc..80193cd546 100644
--- a/doc/bugs/git-annex-fsck___34__-all__34___flag_doesn__39__t_work_for_special_remote.mdwn
+++ b/doc/bugs/git-annex-fsck___34__-all__34___flag_doesn__39__t_work_for_special_remote.mdwn
@@ -24,3 +24,5 @@ I tried to use `git-annex-fsck --all --from remote` to check files on a special
 ### Have you had any luck using git-annex before?
 Yes, it's been very helpful for managing large files between laptops, desktops, external storage, and remote storage.
 
+> Thanks for an excellent test case and a clear bug report. I've fixed this
+> bug. [[done]] --[[Joey]]

From 10703dc817e6ddaf9bd15d8007a8ffa48601def3 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 16 Nov 2016 16:03:23 -0400
Subject: [PATCH 039/367] improve comment

---
 Command/Sync.hs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index acc5fbbc94..85f1f2f2ca 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -172,7 +172,10 @@ mergeConfig :: [Git.Merge.MergeConfig]
 mergeConfig = 
 	[ Git.Merge.MergeNonInteractive
 	-- In several situations, unrelated histories should be merged
-	-- together. This includes pairing in the assistant etc.
+	-- together. This includes pairing in the assistant, and merging
+	-- from a remote into a newly created direct mode repo.
+	-- (Once direct mode is removed, this could be changed, so only
+	-- the assistant uses it.)
 	, Git.Merge.MergeUnrelatedHistories
 	]
 

From c8a0eb03a5636434b25c3998b8c514efb0eb48ae Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 16 Nov 2016 16:17:09 -0400
Subject: [PATCH 040/367] devblog

---
 doc/devblog/day_426__grab_bag.mdwn | 63 ++++++++++++++++++++++++++++++
 1 file changed, 63 insertions(+)
 create mode 100644 doc/devblog/day_426__grab_bag.mdwn

diff --git a/doc/devblog/day_426__grab_bag.mdwn b/doc/devblog/day_426__grab_bag.mdwn
new file mode 100644
index 0000000000..4a3bf908b3
--- /dev/null
+++ b/doc/devblog/day_426__grab_bag.mdwn
@@ -0,0 +1,63 @@
+Fixed one howler of a bug today. Turns out that
+`git annex fsck --all --from remote` didn't actually check the content of
+the remote, but checked the local repository. Only `--all` was buggy;
+`git annex fsck --from remote` was ok. Don't think this is crash priority
+enough to make a release for, since only `--all` is affected.
+
+Somewhat uncomfortably made `git annex sync` pass
+`--allow-unrelated-histories` to git merge. While I do think that git's
+recent refusal to merge unrelated histories is good in general, the
+problem is that initializing a direct mode repository involves making an
+empty commit. So merging from a remote into such a direct mode repository
+means merging unrelated histories, while an indirect mode repository doesn't.
+Seems best to avoid such inconsistencies, and the only way I could see to
+do it is to always use `--allow-unrelated-histories`. May revisit this once
+direct mode is finally removed.
+
+Using the git-annex arm standalone bundle on some WD NAS boxes used to
+work, and then it seems they changed their kernel to use a nonstandard page
+size, and broke it. This actually seems to be a 
+[bug in the gold linker](http://bugs.debian.org/844467), which defaults to an
+unncessarily small page size on arm. The git-annex arm bundle is being
+adjusted to try to deal with this.
+
+ghc 8 made `error` include some backtrace information. While it's really
+nice to have backtraces for unexpected exceptions in Haskell, it turns
+out that git-annex used `error` a lot with the intent of showing an error
+message to the user, and a backtrace clutters up such messages. So,
+bit the bullet and checked through every `error` in git-annex and made such
+ones not include a backtrace.
+
+Also, I've been considering what protocol to use between git-annex nodes
+when communicating over tor. One way would be to make it very similar to
+`git-annex-shell`, using rsync etc, and possibly reusing code from
+git-annex-shell. However, it can take a while to make a connection across
+the tor network, and that method seems to need a new connection for each
+file transfered etc. Also thought about using a http based protocol. The
+servant library is great for that, you get both http client and server
+implementations almost for free. Resuming interrupted transfers might
+complicate it, and the hidden service side would need to listen on a unix
+socket, instead of the regular http port. It might be worth it to use http
+for tor, if it could be reused for git-annex http servers not on the tor
+network. But, then I'd have to make the http server support git pull and
+push over http in a way that's compatable with how git uses http, including
+authentication. Which is a whole nother ball of complexity. So, I'm leaning
+instead to using a simple custom protocol something like:
+
+		> AUTH $localuuid $token
+		< AUTH-OK $remoteuuid
+		> SENDPACK $length
+		> $gitdata
+		< RECVPACK $length
+		< $gitdata
+		> GET $pos $key
+		< SENDING $length
+		< $bytes
+		> GET-OK
+		> PUT $key
+		< SEND $pos
+		> SENDING $length
+		> $bytes
+		< PUT-OK
+
+Today's work was sponsored by Riku Voipio.

From c9d51f6064eaea8ddd16f04021688a6c88afa0ec Mon Sep 17 00:00:00 2001
From: "https://anarc.at/openid/" <anarcat@web>
Date: Wed, 16 Nov 2016 21:58:08 +0000
Subject: [PATCH 041/367] Added a comment: how about reusing the special remote
 protocol?

---
 ...omment_1_4d01c756850032d351fa99188a3301a7._comment | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 doc/devblog/day_426__grab_bag/comment_1_4d01c756850032d351fa99188a3301a7._comment

diff --git a/doc/devblog/day_426__grab_bag/comment_1_4d01c756850032d351fa99188a3301a7._comment b/doc/devblog/day_426__grab_bag/comment_1_4d01c756850032d351fa99188a3301a7._comment
new file mode 100644
index 0000000000..7b5a2949a9
--- /dev/null
+++ b/doc/devblog/day_426__grab_bag/comment_1_4d01c756850032d351fa99188a3301a7._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://anarc.at/openid/"
+ nickname="anarcat"
+ avatar="http://cdn.libravatar.org/avatar/b36dcf65657dd36128161355d8920a99503def9461c1bb212410980fe6f07125"
+ subject="how about reusing the special remote protocol?"
+ date="2016-11-16T21:58:08Z"
+ content="""
+git-annex already has a custom protocol detailed in [[design/external_special_remote_protocol]]. it could be quite useful to have that protocol extended to support direct object transfer instead of having to mess around with temporary files like may remotes do, for example...
+
+maybe that makes no sense at all, i don't know. :) --[[anarcat]]
+"""]]

From aedec5d08d10bcd75940e8c4fa8a5415a89271db Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 16 Nov 2016 18:05:42 -0400
Subject: [PATCH 042/367] arm build uses 32kb page size

(Change was made in gitannexbuilder scripts not here.)
---
 CHANGELOG                                          |  2 ++
 ...ent_8_48026cf7c187e97d53d15d35ed2c3670._comment | 14 ++++++++++++++
 2 files changed, 16 insertions(+)
 create mode 100644 doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS/comment_8_48026cf7c187e97d53d15d35ed2c3670._comment

diff --git a/CHANGELOG b/CHANGELOG
index 6dd4edb3ec..5bd9d3e9d7 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -13,6 +13,8 @@ git-annex (6.20161112) UNRELEASED; urgency=medium
     only use backtraces for unexpected errors.
   * fsck --all --from was checking the existence and content of files
     in the local repository, rather than on the special remote. Oops.
+  * Linux arm standalone: Build with a 32kb page size, which is needed
+    on several ARM NAS devices, including Drobo 5N, and WD NAS.
 
  -- Joey Hess <id@joeyh.name>  Tue, 15 Nov 2016 11:15:27 -0400
 
diff --git a/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS/comment_8_48026cf7c187e97d53d15d35ed2c3670._comment b/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS/comment_8_48026cf7c187e97d53d15d35ed2c3670._comment
new file mode 100644
index 0000000000..493031115a
--- /dev/null
+++ b/doc/bugs/git-annex_won__39__t_execute_on_WD_My_Cloud_NAS/comment_8_48026cf7c187e97d53d15d35ed2c3670._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 8"""
+ date="2016-11-16T21:48:49Z"
+ content="""
+The arm daily build now uses a 32kb page size. So try
+<https://downloads.kitenet.net/git-annex/autobuild/armel/git-annex-standalone-armel.tar.gz>
+
+That has been verified to fix the problem on a Drobo 5N.
+
+This may still not be enough for some of the affected NAS devices, which
+use a 64kb page size. Unfortunately, gold fails to link with a 64kb page
+size: <http://bugs.debian.org/844467>
+"""]]

From 8ac3b30ca02767331a330fa26aa416664b94c793 Mon Sep 17 00:00:00 2001
From: "luckcolorsgoo@ab4f3c1c44a7dbcbcb9d9a29315b59ad524ceaaa"
 <luckcolorsgoo@web>
Date: Wed, 16 Nov 2016 22:56:46 +0000
Subject: [PATCH 043/367] Added a comment

---
 ...nt_2_afa6a131999feda67876859cd85ebcfc._comment | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/bugs/Adding_zip_or_7z_or_tar_archive_builds_for_windows/comment_2_afa6a131999feda67876859cd85ebcfc._comment

diff --git a/doc/bugs/Adding_zip_or_7z_or_tar_archive_builds_for_windows/comment_2_afa6a131999feda67876859cd85ebcfc._comment b/doc/bugs/Adding_zip_or_7z_or_tar_archive_builds_for_windows/comment_2_afa6a131999feda67876859cd85ebcfc._comment
new file mode 100644
index 0000000000..b0db544798
--- /dev/null
+++ b/doc/bugs/Adding_zip_or_7z_or_tar_archive_builds_for_windows/comment_2_afa6a131999feda67876859cd85ebcfc._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="luckcolorsgoo@ab4f3c1c44a7dbcbcb9d9a29315b59ad524ceaaa"
+ nickname="luckcolorsgoo"
+ avatar="http://cdn.libravatar.org/avatar/ddff84cd2a97252a05cccb4bc5010448"
+ subject="comment 2"
+ date="2016-11-16T22:56:46Z"
+ content="""
+In my case i was going to make a script for automatically downloading and updating an git portbale / git annex instance, by first fetching git portbale and then downloading the gitannex exe.
+
+So yeah it's more reliable to extract an archive rather than trying to extract the setup without executing it.
+That's why i'm asking for this feature. :)
+
+
+
+"""]]

From 95b5afb706373ab6891bf05a50c99af8248ab93f Mon Sep 17 00:00:00 2001
From: db48x <db48x@web>
Date: Thu, 17 Nov 2016 04:00:18 +0000
Subject: [PATCH 044/367]

---
 ...ror_message_makes_useless_suggestions.mdwn | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 doc/bugs/when_you_get_a_file_but_don__39__t_actually_have_enough_space_for_it__44___the_error_message_makes_useless_suggestions.mdwn

diff --git a/doc/bugs/when_you_get_a_file_but_don__39__t_actually_have_enough_space_for_it__44___the_error_message_makes_useless_suggestions.mdwn b/doc/bugs/when_you_get_a_file_but_don__39__t_actually_have_enough_space_for_it__44___the_error_message_makes_useless_suggestions.mdwn
new file mode 100644
index 0000000000..19e839263e
--- /dev/null
+++ b/doc/bugs/when_you_get_a_file_but_don__39__t_actually_have_enough_space_for_it__44___the_error_message_makes_useless_suggestions.mdwn
@@ -0,0 +1,21 @@
+The suggestion to make remotes available isn't really applicable, since the error was local.
+
+This is with git annex 6.20161110-gd48f4ca.
+
+[[!format sh """
+  ../git-annex.linux/git-annex get archiveteam-fire/metro.co.uk-urls-2007-04-12-20150627/metro.co.uk-urls-2007-04-12-20150627_meta.xml
+get archiveteam-fire/metro.co.uk-urls-2007-04-12-20150627/metro.co.uk-urls-2007-04-12-20150627_meta.xml 
+  not enough free space, need 98.82 GB more (use --force to override this check or adjust annex.diskreserve)
+
+  Unable to access these remotes: web
+
+  Try making some of these repositories available:
+        00000000-0000-0000-0000-000000000001 -- web
+        9f8218c0-763f-463d-9152-ecdc56d4452c -- iabak@redwyne.jwintjen.de:~/IA.BAK/shard12
+failed
+git-annex: get: 1 failed
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+mixed success

From 50836695b7cde2d7c40a6bd47370fc235c772e92 Mon Sep 17 00:00:00 2001
From: andrew <andrew@web>
Date: Thu, 17 Nov 2016 14:59:15 +0000
Subject: [PATCH 045/367] Added a comment: RESOLVED

---
 ...ent_1_39718e8a35e42421a8aaf3316ae1d76a._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/bugs/DBG__58___running___96____47__Users__47__joey__47__homebrew__47__opt__47__gpg-agent__47__bin__47__gpg-agent__39___for_testing_failed/comment_1_39718e8a35e42421a8aaf3316ae1d76a._comment

diff --git a/doc/bugs/DBG__58___running___96____47__Users__47__joey__47__homebrew__47__opt__47__gpg-agent__47__bin__47__gpg-agent__39___for_testing_failed/comment_1_39718e8a35e42421a8aaf3316ae1d76a._comment b/doc/bugs/DBG__58___running___96____47__Users__47__joey__47__homebrew__47__opt__47__gpg-agent__47__bin__47__gpg-agent__39___for_testing_failed/comment_1_39718e8a35e42421a8aaf3316ae1d76a._comment
new file mode 100644
index 0000000000..02b62f46e8
--- /dev/null
+++ b/doc/bugs/DBG__58___running___96____47__Users__47__joey__47__homebrew__47__opt__47__gpg-agent__47__bin__47__gpg-agent__39___for_testing_failed/comment_1_39718e8a35e42421a8aaf3316ae1d76a._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="RESOLVED"
+ date="2016-11-17T14:59:15Z"
+ content="""
+Ooops. I am on OS-X. I use brew for my gnupg installation. It appears I had removed gpg from the path when installing something. I just needed to run to fix:
+
+    brew link gnupg2
+
+Thanks,
+
+Andrew
+"""]]

From 6a5592c05fa6a0698f854472be171b1ae831d51b Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 17 Nov 2016 13:39:30 -0400
Subject: [PATCH 046/367] avoid tab warnings from ghc 8

---
 ghci | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ghci b/ghci
index b8e4539df8..fccfc661ab 100755
--- a/ghci
+++ b/ghci
@@ -1,4 +1,4 @@
 #!/bin/sh
 # ghci using objects built by cabal
 make dist/caballog
-$(grep 'ghc --make' dist/caballog | head -n 1 | perl -pe 's/--make/--interactive/; s/.\/[^\.\s]+.hs//; s/-package-id [^\s]+//g; s/-hide-all-packages//; s/-threaded//') $@
+$(grep 'ghc --make' dist/caballog | head -n 1 | perl -pe 's/--make/--interactive/; s/.\/[^\.\s]+.hs//; s/-package-id [^\s]+//g; s/-hide-all-packages//; s/-threaded//') $@ -fno-warn-tabs

From 9cc869e5836b94a8be5c93a9d8b902a35a5573ef Mon Sep 17 00:00:00 2001
From: andrew <andrew@web>
Date: Thu, 17 Nov 2016 17:49:28 +0000
Subject: [PATCH 047/367] Added a comment: git annex copy --auto --to cloud
 works

---
 ..._a7f476aeacf88679f25badc78fad886a._comment | 57 +++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100644 doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_4_a7f476aeacf88679f25badc78fad886a._comment

diff --git a/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_4_a7f476aeacf88679f25badc78fad886a._comment b/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_4_a7f476aeacf88679f25badc78fad886a._comment
new file mode 100644
index 0000000000..eec45e333e
--- /dev/null
+++ b/doc/forum/sync_--content__44___fatal_is_outside_repository_errors/comment_4_a7f476aeacf88679f25badc78fad886a._comment
@@ -0,0 +1,57 @@
+[[!comment format=mdwn
+ username="andrew"
+ avatar="http://cdn.libravatar.org/avatar/acc0ece1eedf07dd9631e7d7d343c435"
+ subject="git annex copy --auto --to cloud works"
+ date="2016-11-17T17:49:27Z"
+ content="""
+Yes, only `git annex sync --content` seems to fail. I am using v6 with a mix of unlocked and locked files. I did not know about the --auto flags for copy/get.
+
+* `git annex copy --auto --to cloud` works fine
+* `git annex get --auto --from cloud` works fine
+
+
+*Are there any symlinks in the path to /Users/andrew/notes ? Eg, is /Users a symlink, or /Users/andrew a symlink, or //Users/andrew/notes itself symlinked to elsewhere?*
+
+**No**
+
+*You say it's only failing for some files. Do the filenames that it's failing on contain any non-ascii characters?*
+
+**They seem normal.**
+
+*So, please paste in git annex version and git annex info output.*
+
+    git-annex version: 6.20161110-gd48f4ca
+    build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV FsEvents XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+    remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+    local repository version: 6
+    supported repository versions: 3 5 6
+    upgrade supported from repository versions: 0 1 2 3 4 5
+    operating system: darwin x86_64
+
+    repository mode: indirect
+    trusted repositories: 0
+    semitrusted repositories: 10
+    00000000-0000-0000-0000-000000000001 -- web
+    00000000-0000-0000-0000-000000000002 -- bittorrent
+    22de57a0-c9ca-4bfe-8349-3141b3a87c8f -- Dream Objects [cloud]
+    334791ca-c284-4a87-a233-fc29be00d31a -- [disc_May-2-2015_a]
+    4c57ac0e-b8fe-4b4b-98d3-fb0a1b6b9657 -- MacBook Air [here]
+    6a85150d-6ea2-4ba1-92ce-8f4ef575b8e0 -- prowl MacBook Mini
+    896c3d52-427a-41a1-867c-d18e6740d758 -- disc_May_4_2015_1
+    96391b13-3981-430f-ac3b-6210e3d4e759 -- [disc_May-2-2015_b]
+    b4a41e90-2398-4bba-aaf5-d8f8cd78a5bc -- 2TB USB Drive [usbdrive]
+    e42b223d-ec04-4ad8-bdf7-8429a45d844c -- disc_May-2-2015_a
+    untrusted repositories: 0
+    transfers in progress: none
+    available local disk space: 2.32 gigabytes (+1 megabyte reserved)
+    temporary object directory size: 29.47 megabytes (clean up with git-annex unused)
+    local annex keys: 4104
+    local annex size: 10.53 gigabytes
+    annexed files in working tree: 6417
+    size of annexed files in working tree: 80.75 gigabytes
+    bloom filter size: 32 mebibytes (0.8% full)
+    backend usage: 
+    SHA256E: 6417
+
+"""]]

From f3ea75a74c8e959d0a82334fa6379dc1f68c9181 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 17 Nov 2016 17:19:53 -0400
Subject: [PATCH 048/367] devblog

---
 doc/devblog/day_426__grab_bag.mdwn | 12 +++----
 doc/devblog/day_427__free_p2p.mdwn | 51 ++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+), 6 deletions(-)
 create mode 100644 doc/devblog/day_427__free_p2p.mdwn

diff --git a/doc/devblog/day_426__grab_bag.mdwn b/doc/devblog/day_426__grab_bag.mdwn
index 4a3bf908b3..36e32077ee 100644
--- a/doc/devblog/day_426__grab_bag.mdwn
+++ b/doc/devblog/day_426__grab_bag.mdwn
@@ -45,19 +45,19 @@ authentication. Which is a whole nother ball of complexity. So, I'm leaning
 instead to using a simple custom protocol something like:
 
 		> AUTH $localuuid $token
-		< AUTH-OK $remoteuuid
+		< AUTH-SUCCESS $remoteuuid
 		> SENDPACK $length
 		> $gitdata
 		< RECVPACK $length
 		< $gitdata
 		> GET $pos $key
-		< SENDING $length
+		< DATA $length
 		< $bytes
-		> GET-OK
+		> SUCCESS
 		> PUT $key
-		< SEND $pos
-		> SENDING $length
+		< PUT-FROM $pos
+		> DATA $length
 		> $bytes
-		< PUT-OK
+		< SUCCESS
 
 Today's work was sponsored by Riku Voipio.
diff --git a/doc/devblog/day_427__free_p2p.mdwn b/doc/devblog/day_427__free_p2p.mdwn
new file mode 100644
index 0000000000..7c727587b1
--- /dev/null
+++ b/doc/devblog/day_427__free_p2p.mdwn
@@ -0,0 +1,51 @@
+For a Haskell programmer, and day where a big thing is implemented
+without the least scrap of code that touches the IO monad is a good day.
+And this was a good day for me!
+
+Implemented the p2p protocol for tor hidden services. Its needs are somewhat
+similar to the external special remote protocol, but the two protocols are
+not fully overlapping with one-another. Rather than try to unify them, and
+so complicate both cases, I prefer to reuse as much code as possible between
+separate protocol implementations. The generating and parsing of messages
+is largely shared between them. I let the new p2p protocol otherwise
+develop in its own direction.
+
+But, I *do* want to make this p2p protocol reusable for other types of p2p
+networks than tor hidden services. This was an opportunity to use the Free
+monad, which I'd never used before. It worked out great, letting me write
+monadic code to handle requests and responses in the protocol, that reads
+the content of files and resumes transfers and so on, all independent
+of any concrete implementation.
+
+The whole implementation of the protocol only needed 74 lines of monadic code.
+It helped that I was able to factor out functions like this one, that is used
+both for handling a download, and by the remote when an upload is sent to it:
+
+	receiveContent :: Key -> Offset -> Len -> Proto Bool
+	receiveContent key offset len = do
+	        content <- receiveBytes len
+	        ok <- writeKeyFile key offset content
+	        sendMessage $ if ok then SUCCESS else FAILURE
+	        return ok
+
+To get transcripts of the protocol in action, the Free monad can be evaluated
+purely, providing the other side of the conversation:
+
+	ghci> putStrLn $ protoDump $ runPure (put (fromJust $ file2key "WORM--foo")) [PUT_FROM (Offset 10), SUCCESS]
+	> PUT WORM--foo
+	< PUT-FROM 10
+	> DATA 90
+	> bytes
+	< SUCCESS
+	result: True
+
+	ghci> putStrLn $ protoDump $ runPure (serve (toUUID "myuuid")) [GET (Offset 0) (fromJust $ file2key "WORM--foo")]
+	< GET 0 WORM--foo
+	> PROTO-ERROR must AUTH first
+	result: ()
+
+Am very happy with all this pure code and that I'm finally using Free monads.
+Next I need to get down the the dirty business of wiring this up to
+actual IO actions, and an actual network connection.
+
+Today's work was sponsored by Jake Vosloo on Patreon.

From 65e903397c75b81278e8a4e5948b9499b2dfb58c Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 17 Nov 2016 17:19:04 -0400
Subject: [PATCH 049/367] implementation of peer-to-peer protocol

For use with tor hidden services, and perhaps other transports later.

Based on Utility.SimpleProtocol, it's a line-based protocol,
interspersed with transfers of bytestrings of a specified size.

Implementation of the local and remote sides of the protocol is done
using a free monad. This lets monadic code be included here, without
tying it to any particular way to get bytes peer-to-peer.

This adds a dependency on the haskell package "free", although that
was probably pulled in transitively from other dependencies already.

This commit was sponsored by Jeff Goeke-Smith on Patreon.
---
 Remote/External/Types.hs  |   8 --
 Remote/Helper/P2P.hs      | 247 ++++++++++++++++++++++++++++++++++++++
 RemoteDaemon/Types.hs     |   4 -
 Types/Key.hs              |   5 +
 Types/UUID.hs             |   6 +
 Utility/SimpleProtocol.hs |   7 ++
 debian/control            |   1 +
 git-annex.cabal           |   2 +
 8 files changed, 268 insertions(+), 12 deletions(-)
 create mode 100644 Remote/Helper/P2P.hs

diff --git a/Remote/External/Types.hs b/Remote/External/Types.hs
index 2306989bb9..ef8724ee79 100644
--- a/Remote/External/Types.hs
+++ b/Remote/External/Types.hs
@@ -250,14 +250,6 @@ instance Proto.Serializable Direction where
 	deserialize "RETRIEVE" = Just Download
 	deserialize _ = Nothing
 
-instance Proto.Serializable Key where
-	serialize = key2file
-	deserialize = file2key
-
-instance Proto.Serializable [Char] where
-	serialize = id
-	deserialize = Just
-
 instance Proto.Serializable ProtocolVersion where
 	serialize = show
 	deserialize = readish
diff --git a/Remote/Helper/P2P.hs b/Remote/Helper/P2P.hs
new file mode 100644
index 0000000000..d973880f74
--- /dev/null
+++ b/Remote/Helper/P2P.hs
@@ -0,0 +1,247 @@
+{- P2P protocol
+ -
+ - Copyright 2016 Joey Hess <id@joeyh.name>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE DeriveFunctor, TemplateHaskell, FlexibleContexts #-}
+
+module Remote.Helper.P2P (
+	AuthToken(..),
+	ProtoF(..),
+	runPure,
+	protoDump,
+	auth,
+	get,
+	put,
+	serve,
+) where
+
+import qualified Utility.SimpleProtocol as Proto
+import Types.Key
+import Types.UUID
+import Utility.Applicative
+import Utility.PartialPrelude
+
+import Control.Monad
+import Control.Monad.Free
+import Control.Monad.Free.TH
+import qualified Data.ByteString.Lazy as L
+
+newtype AuthToken = AuthToken String
+	deriving (Show)
+
+newtype Offset = Offset Integer
+	deriving (Show)
+
+newtype Len = Len Integer
+	deriving (Show)
+
+-- | Messages in the protocol. The peer that makes the connection
+-- always initiates requests, and the other peer makes responses to them.
+data Message
+	= AUTH UUID AuthToken -- uuid of the peer that is authenticating
+	| AUTH_SUCCESS UUID -- uuid of the remote peer
+	| AUTH_FAILURE
+	| GET Offset Key
+	| PUT Key
+	| PUT_FROM Offset
+	| SUCCESS
+	| FAILURE
+	| DATA Len -- followed by bytes
+	| PROTO_ERROR String
+	deriving (Show)
+
+-- | Free monad for implementing actions that use the protocol.
+data ProtoF next
+	= SendMessage Message next
+	| GetMessage (Message -> next)
+	| SendBytes Len L.ByteString next
+	| ReceiveBytes Len (L.ByteString -> next)
+	| KeyFileSize Key (Len -> next)
+	-- ^ Checks size of key file (dne = 0)
+	| ReadKeyFile Key Offset (L.ByteString -> next)
+	| WriteKeyFile Key Offset L.ByteString (Bool -> next)
+	| CheckAuthToken UUID AuthToken (Bool -> next)
+	| SetPresent Key UUID next
+	deriving (Functor)
+
+type Proto = Free ProtoF
+
+$(makeFree ''ProtoF)
+
+-- | Running Proto actions purely, to see what they do.
+runPure :: Show r => Proto r -> [Message] -> [(String, Maybe Message)]
+runPure (Pure r) _ = [("result: " ++ show r, Nothing)]
+runPure (Free (SendMessage m next)) ms = (">",  Just m):runPure next ms
+runPure (Free (GetMessage _)) [] = [("not enough Messages provided", Nothing)]
+runPure (Free (GetMessage next)) (m:ms) = ("<", Just m):runPure (next m) ms
+runPure (Free (SendBytes _ _ next)) ms = ("> bytes", Nothing):runPure next ms
+runPure (Free (ReceiveBytes _ next)) ms = ("< bytes", Nothing):runPure (next L.empty) ms
+runPure (Free (KeyFileSize _ next)) ms = runPure (next (Len 100)) ms
+runPure (Free (ReadKeyFile _ _ next)) ms = runPure (next L.empty) ms
+runPure (Free (WriteKeyFile _ _ _ next)) ms = runPure (next True) ms
+runPure (Free (CheckAuthToken _ _ next)) ms = runPure (next True) ms
+runPure (Free (SetPresent _ _ next)) ms = runPure next ms
+
+protoDump :: [(String, Maybe Message)] -> String
+protoDump = unlines . map protoDump'
+
+protoDump' :: (String, Maybe Message) -> String
+protoDump' (s, Nothing) = s
+protoDump' (s, Just m) = s ++ " " ++ unwords (Proto.formatMessage m)
+
+auth :: UUID -> AuthToken -> Proto (Maybe UUID)
+auth myuuid t = do
+	sendMessage (AUTH myuuid t)
+	r <- getMessage
+	case r of
+		AUTH_SUCCESS theiruuid -> return $ Just theiruuid
+		AUTH_FAILURE -> return Nothing
+		_ -> do
+			sendMessage (PROTO_ERROR "auth failed")
+			return Nothing
+
+get :: Key -> Proto Bool
+get key = do
+	Len n <- keyFileSize key
+	let offset = Offset n
+	sendMessage (GET offset key)
+	r <- getMessage
+	case r of
+		DATA len -> receiveContent key offset len
+		_ -> do
+			sendMessage (PROTO_ERROR "expected DATA")
+			return False
+
+put :: Key -> Proto Bool
+put key = do
+	sendMessage (PUT key)
+	r <- getMessage
+	case r of
+		PUT_FROM offset -> sendContent key offset
+		_ -> do
+			sendMessage (PROTO_ERROR "expected PUT_FROM")
+			return False
+
+-- | Serve the protocol.
+--
+-- Note that if the client sends an unexpected message, the server will
+-- respond with PTOTO_ERROR, and always continues processing messages.
+-- Since the protocol is not versioned, this is necessary to handle
+-- protocol changes robustly, since the client can detect when it's
+-- talking to a server that does not support some new feature, and fall
+-- back.
+--
+-- When the client sends PROTO_ERROR to the server, the server gives up,
+-- since it's not clear what state the client is is, and so not possible to
+-- recover.
+serve :: UUID -> Proto ()
+serve myuuid = go Nothing
+  where
+	go autheduuid = do
+		r <- getMessage
+		case r of
+			AUTH theiruuid authtoken -> do
+				ok <- checkAuthToken theiruuid authtoken
+				if ok
+					then do
+						sendMessage (AUTH_SUCCESS myuuid)
+						go (Just theiruuid)
+					else do
+						sendMessage AUTH_FAILURE
+						go autheduuid
+			PROTO_ERROR _ -> return ()
+			_ -> do
+				case autheduuid of
+					Just theiruuid -> authed theiruuid r
+				Nothing -> sendMessage (PROTO_ERROR "must AUTH first")
+				go autheduuid
+	
+	authed theiruuid r = case r of
+		GET offset key -> do
+			ok <- sendContent key offset
+			when ok $
+				setPresent key theiruuid
+		PUT key -> do
+			(Len n) <- keyFileSize key
+			let offset = Offset n
+			sendMessage (PUT_FROM offset)
+			r' <- getMessage
+			case r' of
+				DATA len -> do
+					void $ receiveContent key offset len
+					setPresent key myuuid
+				_ -> sendMessage (PROTO_ERROR "expected DATA")
+		_ -> sendMessage (PROTO_ERROR "unexpected command")
+
+sendContent :: Key -> Offset -> Proto Bool
+sendContent key offset = do
+	(len, content) <- readKeyFile' key offset
+	sendMessage (DATA len)
+	sendBytes len content
+	ack <- getMessage
+	case ack of
+		SUCCESS -> return True
+		FAILURE -> return False
+		_ -> do
+			sendMessage (PROTO_ERROR "expected SUCCESS or FAILURE")
+			return False
+
+receiveContent :: Key -> Offset -> Len -> Proto Bool
+receiveContent key offset len = do
+	content <- receiveBytes len
+	ok <- writeKeyFile key offset content
+	sendMessage $ if ok then SUCCESS else FAILURE
+	return ok
+
+-- Reads key file from an offset. The Len should correspond to
+-- the length of the ByteString, but to avoid buffering the content
+-- in memory, is gotten using keyFileSize.
+readKeyFile' :: Key -> Offset -> Proto (Len, L.ByteString)
+readKeyFile' key (Offset offset) = do
+	(Len totallen) <- keyFileSize key
+	let len = totallen - offset
+	if len <= 0
+		then return (Len 0, L.empty)
+		else do
+			content <- readKeyFile key (Offset offset)
+			return (Len len, content)
+
+instance Proto.Sendable Message where
+	formatMessage (AUTH uuid authtoken) = ["AUTH", Proto.serialize uuid, Proto.serialize authtoken]
+	formatMessage (AUTH_SUCCESS uuid) = ["AUTH-SUCCESS",  Proto.serialize uuid]
+	formatMessage AUTH_FAILURE = ["AUTH-FAILURE"]
+	formatMessage (GET offset key) = ["GET", Proto.serialize offset, Proto.serialize key]
+	formatMessage (PUT key) = ["PUT", Proto.serialize key]
+	formatMessage (PUT_FROM offset) = ["PUT-FROM", Proto.serialize offset]
+	formatMessage SUCCESS = ["SUCCESS"]
+	formatMessage FAILURE = ["FAILURE"]
+	formatMessage (DATA leng) = ["DATA", Proto.serialize leng]
+	formatMessage (PROTO_ERROR err) = ["PROTO-ERROR", Proto.serialize err]
+
+instance Proto.Receivable Message where
+	parseCommand "AUTH" = Proto.parse2 AUTH
+	parseCommand "AUTH-SUCCESS" = Proto.parse1 AUTH_SUCCESS
+	parseCommand "AUTH-FAILURE" = Proto.parse0 AUTH_FAILURE
+	parseCommand "GET" = Proto.parse2 GET
+	parseCommand "PUT" = Proto.parse1 PUT
+	parseCommand "PUT-FROM" = Proto.parse1 PUT_FROM
+	parseCommand "SUCCESS" = Proto.parse0 SUCCESS
+	parseCommand "FAILURE" = Proto.parse0 FAILURE
+	parseCommand "DATA" = Proto.parse1 DATA
+	parseCommand "PROTO-ERROR" = Proto.parse1 PROTO_ERROR
+	parseCommand _ = Proto.parseFail
+
+instance Proto.Serializable Offset where
+	serialize (Offset n) = show n
+	deserialize = Offset <$$> readish
+
+instance Proto.Serializable Len where
+	serialize (Len n) = show n
+	deserialize = Len <$$> readish
+
+instance Proto.Serializable AuthToken where
+	serialize (AuthToken s) = s
+	deserialize = Just . AuthToken
diff --git a/RemoteDaemon/Types.hs b/RemoteDaemon/Types.hs
index f85219ea5e..ba88aa685b 100644
--- a/RemoteDaemon/Types.hs
+++ b/RemoteDaemon/Types.hs
@@ -100,10 +100,6 @@ instance Proto.Serializable RemoteURI where
 	serialize (RemoteURI u) = show u
 	deserialize = RemoteURI <$$> parseURI
 
-instance Proto.Serializable [Char] where
-	serialize = id
-	deserialize = Just
-
 instance Proto.Serializable RefList where
 	serialize = unwords . map Git.fromRef
 	deserialize = Just . map Git.Ref . words
diff --git a/Types/Key.hs b/Types/Key.hs
index 3642eca1c1..598fe43cc8 100644
--- a/Types/Key.hs
+++ b/Types/Key.hs
@@ -27,6 +27,7 @@ import qualified Data.Text as T
 import Common
 import Utility.QuickCheck
 import Utility.Bloom
+import qualified Utility.SimpleProtocol as Proto
 
 {- A Key has a unique name, which is derived from a particular backend,
  - and may contain other optional metadata. -}
@@ -129,6 +130,10 @@ instance FromJSON Key where
 	parseJSON (String t) = maybe mempty pure $ file2key $ T.unpack t
 	parseJSON _ = mempty
 
+instance Proto.Serializable Key where
+	serialize = key2file
+	deserialize = file2key
+
 instance Arbitrary Key where
 	arbitrary = Key
 		<$> (listOf1 $ elements $ ['A'..'Z'] ++ ['a'..'z'] ++ ['0'..'9'] ++ "-_\r\n \t")
diff --git a/Types/UUID.hs b/Types/UUID.hs
index 4212eaa7f9..f5c9cda301 100644
--- a/Types/UUID.hs
+++ b/Types/UUID.hs
@@ -13,6 +13,8 @@ import qualified Data.Map as M
 import qualified Data.UUID as U
 import Data.Maybe
 
+import qualified Utility.SimpleProtocol as Proto
+
 -- A UUID is either an arbitrary opaque string, or UUID info may be missing.
 data UUID = NoUUID | UUID String
 	deriving (Eq, Ord, Show, Read)
@@ -35,3 +37,7 @@ isUUID :: String -> Bool
 isUUID = isJust . U.fromString
 
 type UUIDMap = M.Map UUID String
+
+instance Proto.Serializable UUID where
+	serialize = fromUUID
+	deserialize = Just . toUUID
diff --git a/Utility/SimpleProtocol.hs b/Utility/SimpleProtocol.hs
index 708f590e72..728b135e8d 100644
--- a/Utility/SimpleProtocol.hs
+++ b/Utility/SimpleProtocol.hs
@@ -5,6 +5,9 @@
  - License: BSD-2-clause
  -}
 
+{-# LANGUAGE FlexibleInstances #-}
+{-# OPTIONS_GHC -fno-warn-orphans #-}
+
 module Utility.SimpleProtocol (
 	Sendable(..),
 	Receivable(..),
@@ -88,3 +91,7 @@ dupIoHandles = do
 	nullh `hDuplicateTo` stdin
 	stderr `hDuplicateTo` stdout
 	return (readh, writeh)
+
+instance Serializable [Char] where
+	serialize = id
+	deserialize = Just
diff --git a/debian/control b/debian/control
index ec77a2946e..07630dfa21 100644
--- a/debian/control
+++ b/debian/control
@@ -64,6 +64,7 @@ Build-Depends:
 	libghc-xml-types-dev,
 	libghc-async-dev,
 	libghc-monad-logger-dev,
+	ligghc-free-dev,
 	libghc-feed-dev (>= 0.3.9.2),
 	libghc-regex-tdfa-dev,
 	libghc-tasty-dev (>= 0.7),
diff --git a/git-annex.cabal b/git-annex.cabal
index eb819463bd..1356536906 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -342,6 +342,7 @@ Executable git-annex
    MissingH,
    hslogger,
    monad-logger,
+   free,
    utf8-string,
    bytestring,
    text,
@@ -918,6 +919,7 @@ Executable git-annex
     Remote.Helper.Hooks
     Remote.Helper.Http
     Remote.Helper.Messages
+    Remote.Helper.P2P
     Remote.Helper.ReadOnly
     Remote.Helper.Special
     Remote.Helper.Ssh

From ae403be24b70208458ef7d770150a53a6ae909ed Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 17 Nov 2016 20:54:14 -0400
Subject: [PATCH 050/367] avoid setPresent when sending to a peer

This mirrors how git-annex-shell works; recvKey updates location
tracking, but sendKey does not.
---
 Remote/Helper/P2P.hs | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/Remote/Helper/P2P.hs b/Remote/Helper/P2P.hs
index d973880f74..0b4d05847e 100644
--- a/Remote/Helper/P2P.hs
+++ b/Remote/Helper/P2P.hs
@@ -160,10 +160,6 @@ serve myuuid = go Nothing
 				go autheduuid
 	
 	authed theiruuid r = case r of
-		GET offset key -> do
-			ok <- sendContent key offset
-			when ok $
-				setPresent key theiruuid
 		PUT key -> do
 			(Len n) <- keyFileSize key
 			let offset = Offset n
@@ -174,6 +170,9 @@ serve myuuid = go Nothing
 					void $ receiveContent key offset len
 					setPresent key myuuid
 				_ -> sendMessage (PROTO_ERROR "expected DATA")
+		-- setPresent not called because the peer may have
+		-- requested the data but not permanatly stored it.
+		GET offset key -> sendContent key offset
 		_ -> sendMessage (PROTO_ERROR "unexpected command")
 
 sendContent :: Key -> Offset -> Proto Bool

From 505d1df8ab543f01714901d2e1f57e868d69f58c Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 17 Nov 2016 21:04:35 -0400
Subject: [PATCH 051/367] refactor

---
 Remote/Helper/P2P.hs | 50 +++++++++++++++++++-------------------------
 1 file changed, 22 insertions(+), 28 deletions(-)

diff --git a/Remote/Helper/P2P.hs b/Remote/Helper/P2P.hs
index 0b4d05847e..bf25a4ed9f 100644
--- a/Remote/Helper/P2P.hs
+++ b/Remote/Helper/P2P.hs
@@ -104,16 +104,7 @@ auth myuuid t = do
 			return Nothing
 
 get :: Key -> Proto Bool
-get key = do
-	Len n <- keyFileSize key
-	let offset = Offset n
-	sendMessage (GET offset key)
-	r <- getMessage
-	case r of
-		DATA len -> receiveContent key offset len
-		_ -> do
-			sendMessage (PROTO_ERROR "expected DATA")
-			return False
+get key = receiveContent key (`GET` key)
 
 put :: Key -> Proto Bool
 put key = do
@@ -156,23 +147,17 @@ serve myuuid = go Nothing
 			_ -> do
 				case autheduuid of
 					Just theiruuid -> authed theiruuid r
-				Nothing -> sendMessage (PROTO_ERROR "must AUTH first")
+					Nothing -> sendMessage (PROTO_ERROR "must AUTH first")
 				go autheduuid
 	
-	authed theiruuid r = case r of
+	authed _theiruuid r = case r of
 		PUT key -> do
-			(Len n) <- keyFileSize key
-			let offset = Offset n
-			sendMessage (PUT_FROM offset)
-			r' <- getMessage
-			case r' of
-				DATA len -> do
-					void $ receiveContent key offset len
-					setPresent key myuuid
-				_ -> sendMessage (PROTO_ERROR "expected DATA")
+			ok <- receiveContent key PUT_FROM
+			when ok $
+				setPresent key myuuid
 		-- setPresent not called because the peer may have
 		-- requested the data but not permanatly stored it.
-		GET offset key -> sendContent key offset
+		GET offset key -> void $ sendContent key offset
 		_ -> sendMessage (PROTO_ERROR "unexpected command")
 
 sendContent :: Key -> Offset -> Proto Bool
@@ -188,12 +173,21 @@ sendContent key offset = do
 			sendMessage (PROTO_ERROR "expected SUCCESS or FAILURE")
 			return False
 
-receiveContent :: Key -> Offset -> Len -> Proto Bool
-receiveContent key offset len = do
-	content <- receiveBytes len
-	ok <- writeKeyFile key offset content
-	sendMessage $ if ok then SUCCESS else FAILURE
-	return ok
+receiveContent :: Key -> (Offset -> Message) -> Proto Bool
+receiveContent key mkmsg = do
+	Len n <- keyFileSize key
+	let offset = Offset n
+	sendMessage (mkmsg offset)
+	r <- getMessage
+	case r of
+		DATA len -> do
+			content <- receiveBytes len
+			ok <- writeKeyFile key offset content
+			sendMessage $ if ok then SUCCESS else FAILURE
+			return ok
+		_ -> do
+			sendMessage (PROTO_ERROR "expected DATA")
+			return False
 
 -- Reads key file from an offset. The Len should correspond to
 -- the length of the ByteString, but to avoid buffering the content

From 47b7028d7cf75c9325ddb059f64fa4c434a804a9 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 17 Nov 2016 21:27:16 -0400
Subject: [PATCH 052/367] pass Len to writeKeyFile so it can detect short reads

---
 Remote/Helper/P2P.hs | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/Remote/Helper/P2P.hs b/Remote/Helper/P2P.hs
index bf25a4ed9f..b94eda8503 100644
--- a/Remote/Helper/P2P.hs
+++ b/Remote/Helper/P2P.hs
@@ -59,10 +59,19 @@ data ProtoF next
 	| GetMessage (Message -> next)
 	| SendBytes Len L.ByteString next
 	| ReceiveBytes Len (L.ByteString -> next)
+	-- ^ Lazily reads bytes from peer. Stops once Len are read,
+	-- or if connection is lost, and in either case returns the bytes
+	-- that were read. This allows resuming interrupted transfers.
 	| KeyFileSize Key (Len -> next)
 	-- ^ Checks size of key file (dne = 0)
 	| ReadKeyFile Key Offset (L.ByteString -> next)
-	| WriteKeyFile Key Offset L.ByteString (Bool -> next)
+	| WriteKeyFile Key Offset Len L.ByteString (Bool -> next)
+	-- ^ Writes to key file starting at an offset. Returns True
+	-- once the whole content of the key is stored in the key file.
+	--
+	-- Note: The ByteString may not contain the entire remaining content
+	-- of the key. Only once the key file size == Len has the whole
+	-- content been transferred.
 	| CheckAuthToken UUID AuthToken (Bool -> next)
 	| SetPresent Key UUID next
 	deriving (Functor)
@@ -81,7 +90,7 @@ runPure (Free (SendBytes _ _ next)) ms = ("> bytes", Nothing):runPure next ms
 runPure (Free (ReceiveBytes _ next)) ms = ("< bytes", Nothing):runPure (next L.empty) ms
 runPure (Free (KeyFileSize _ next)) ms = runPure (next (Len 100)) ms
 runPure (Free (ReadKeyFile _ _ next)) ms = runPure (next L.empty) ms
-runPure (Free (WriteKeyFile _ _ _ next)) ms = runPure (next True) ms
+runPure (Free (WriteKeyFile _ _ _ _ next)) ms = runPure (next True) ms
 runPure (Free (CheckAuthToken _ _ next)) ms = runPure (next True) ms
 runPure (Free (SetPresent _ _ next)) ms = runPure next ms
 
@@ -181,8 +190,7 @@ receiveContent key mkmsg = do
 	r <- getMessage
 	case r of
 		DATA len -> do
-			content <- receiveBytes len
-			ok <- writeKeyFile key offset content
+			ok <- writeKeyFile key offset len =<< receiveBytes len
 			sendMessage $ if ok then SUCCESS else FAILURE
 			return ok
 		_ -> do

From 2b33452bd8e4c3ba0d1b36b5d4c6571e47d5bca2 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 17 Nov 2016 21:37:49 -0400
Subject: [PATCH 053/367] add ALREADY-HAVE response to PUT

---
 Remote/Helper/P2P.hs | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/Remote/Helper/P2P.hs b/Remote/Helper/P2P.hs
index b94eda8503..985f524f0d 100644
--- a/Remote/Helper/P2P.hs
+++ b/Remote/Helper/P2P.hs
@@ -47,6 +47,7 @@ data Message
 	| GET Offset Key
 	| PUT Key
 	| PUT_FROM Offset
+	| ALREADY_HAVE
 	| SUCCESS
 	| FAILURE
 	| DATA Len -- followed by bytes
@@ -74,6 +75,8 @@ data ProtoF next
 	-- content been transferred.
 	| CheckAuthToken UUID AuthToken (Bool -> next)
 	| SetPresent Key UUID next
+	| CheckPresent Key (Bool -> next)
+	-- ^ Checks if the whole content of the key is locally present.
 	deriving (Functor)
 
 type Proto = Free ProtoF
@@ -93,6 +96,7 @@ runPure (Free (ReadKeyFile _ _ next)) ms = runPure (next L.empty) ms
 runPure (Free (WriteKeyFile _ _ _ _ next)) ms = runPure (next True) ms
 runPure (Free (CheckAuthToken _ _ next)) ms = runPure (next True) ms
 runPure (Free (SetPresent _ _ next)) ms = runPure next ms
+runPure (Free (CheckPresent _ next)) ms = runPure (next False) ms
 
 protoDump :: [(String, Maybe Message)] -> String
 protoDump = unlines . map protoDump'
@@ -121,6 +125,7 @@ put key = do
 	r <- getMessage
 	case r of
 		PUT_FROM offset -> sendContent key offset
+		ALREADY_HAVE -> return True
 		_ -> do
 			sendMessage (PROTO_ERROR "expected PUT_FROM")
 			return False
@@ -161,9 +166,13 @@ serve myuuid = go Nothing
 	
 	authed _theiruuid r = case r of
 		PUT key -> do
-			ok <- receiveContent key PUT_FROM
-			when ok $
-				setPresent key myuuid
+			have <- checkPresent key
+			if have
+				then sendMessage ALREADY_HAVE
+				else do
+					ok <- receiveContent key PUT_FROM
+					when ok $
+						setPresent key myuuid
 		-- setPresent not called because the peer may have
 		-- requested the data but not permanatly stored it.
 		GET offset key -> void $ sendContent key offset
@@ -217,6 +226,7 @@ instance Proto.Sendable Message where
 	formatMessage (GET offset key) = ["GET", Proto.serialize offset, Proto.serialize key]
 	formatMessage (PUT key) = ["PUT", Proto.serialize key]
 	formatMessage (PUT_FROM offset) = ["PUT-FROM", Proto.serialize offset]
+	formatMessage ALREADY_HAVE = ["ALREADY-HAVE"]
 	formatMessage SUCCESS = ["SUCCESS"]
 	formatMessage FAILURE = ["FAILURE"]
 	formatMessage (DATA leng) = ["DATA", Proto.serialize leng]
@@ -229,6 +239,7 @@ instance Proto.Receivable Message where
 	parseCommand "GET" = Proto.parse2 GET
 	parseCommand "PUT" = Proto.parse1 PUT
 	parseCommand "PUT-FROM" = Proto.parse1 PUT_FROM
+	parseCommand "ALREADY-HAVE" = Proto.parse0 ALREADY_HAVE
 	parseCommand "SUCCESS" = Proto.parse0 SUCCESS
 	parseCommand "FAILURE" = Proto.parse0 FAILURE
 	parseCommand "DATA" = Proto.parse1 DATA

From cbffb61083e1f3aeaee246a000be873bcc24e5c2 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 17 Nov 2016 21:48:59 -0400
Subject: [PATCH 054/367] added REMOVE to protocol

---
 Remote/Helper/P2P.hs | 34 +++++++++++++++++++++++++++-------
 1 file changed, 27 insertions(+), 7 deletions(-)

diff --git a/Remote/Helper/P2P.hs b/Remote/Helper/P2P.hs
index 985f524f0d..19647e5d90 100644
--- a/Remote/Helper/P2P.hs
+++ b/Remote/Helper/P2P.hs
@@ -13,6 +13,7 @@ module Remote.Helper.P2P (
 	runPure,
 	protoDump,
 	auth,
+	remove,
 	get,
 	put,
 	serve,
@@ -44,6 +45,7 @@ data Message
 	= AUTH UUID AuthToken -- uuid of the peer that is authenticating
 	| AUTH_SUCCESS UUID -- uuid of the remote peer
 	| AUTH_FAILURE
+	| REMOVE Key
 	| GET Offset Key
 	| PUT Key
 	| PUT_FROM Offset
@@ -77,6 +79,9 @@ data ProtoF next
 	| SetPresent Key UUID next
 	| CheckPresent Key (Bool -> next)
 	-- ^ Checks if the whole content of the key is locally present.
+	| RemoveKeyFile Key (Bool -> next)
+	-- ^ If the key file is not present, still succeeds.
+	-- May fail if not enough copies to safely drop, etc.
 	deriving (Functor)
 
 type Proto = Free ProtoF
@@ -97,6 +102,7 @@ runPure (Free (WriteKeyFile _ _ _ _ next)) ms = runPure (next True) ms
 runPure (Free (CheckAuthToken _ _ next)) ms = runPure (next True) ms
 runPure (Free (SetPresent _ _ next)) ms = runPure next ms
 runPure (Free (CheckPresent _ next)) ms = runPure (next False) ms
+runPure (Free (RemoveKeyFile _ next)) ms = runPure (next False) ms
 
 protoDump :: [(String, Maybe Message)] -> String
 protoDump = unlines . map protoDump'
@@ -116,6 +122,11 @@ auth myuuid t = do
 			sendMessage (PROTO_ERROR "auth failed")
 			return Nothing
 
+remove :: Key -> Proto Bool
+remove key = do
+	sendMessage (REMOVE key)
+	checkSuccess
+
 get :: Key -> Proto Bool
 get key = receiveContent key (`GET` key)
 
@@ -165,6 +176,9 @@ serve myuuid = go Nothing
 				go autheduuid
 	
 	authed _theiruuid r = case r of
+		REMOVE key -> do
+			ok <- removeKeyFile key
+			sendMessage $ if ok then SUCCESS else FAILURE
 		PUT key -> do
 			have <- checkPresent key
 			if have
@@ -183,13 +197,7 @@ sendContent key offset = do
 	(len, content) <- readKeyFile' key offset
 	sendMessage (DATA len)
 	sendBytes len content
-	ack <- getMessage
-	case ack of
-		SUCCESS -> return True
-		FAILURE -> return False
-		_ -> do
-			sendMessage (PROTO_ERROR "expected SUCCESS or FAILURE")
-			return False
+	checkSuccess
 
 receiveContent :: Key -> (Offset -> Message) -> Proto Bool
 receiveContent key mkmsg = do
@@ -206,6 +214,16 @@ receiveContent key mkmsg = do
 			sendMessage (PROTO_ERROR "expected DATA")
 			return False
 
+checkSuccess :: Proto Bool
+checkSuccess = do
+	ack <- getMessage
+	case ack of
+		SUCCESS -> return True
+		FAILURE -> return False
+		_ -> do
+			sendMessage (PROTO_ERROR "expected SUCCESS or FAILURE")
+			return False
+
 -- Reads key file from an offset. The Len should correspond to
 -- the length of the ByteString, but to avoid buffering the content
 -- in memory, is gotten using keyFileSize.
@@ -223,6 +241,7 @@ instance Proto.Sendable Message where
 	formatMessage (AUTH uuid authtoken) = ["AUTH", Proto.serialize uuid, Proto.serialize authtoken]
 	formatMessage (AUTH_SUCCESS uuid) = ["AUTH-SUCCESS",  Proto.serialize uuid]
 	formatMessage AUTH_FAILURE = ["AUTH-FAILURE"]
+	formatMessage (REMOVE key) = ["REMOVE", Proto.serialize key]
 	formatMessage (GET offset key) = ["GET", Proto.serialize offset, Proto.serialize key]
 	formatMessage (PUT key) = ["PUT", Proto.serialize key]
 	formatMessage (PUT_FROM offset) = ["PUT-FROM", Proto.serialize offset]
@@ -236,6 +255,7 @@ instance Proto.Receivable Message where
 	parseCommand "AUTH" = Proto.parse2 AUTH
 	parseCommand "AUTH-SUCCESS" = Proto.parse1 AUTH_SUCCESS
 	parseCommand "AUTH-FAILURE" = Proto.parse0 AUTH_FAILURE
+	parseCommand "REMOVE" = Proto.parse1 REMOVE
 	parseCommand "GET" = Proto.parse2 GET
 	parseCommand "PUT" = Proto.parse1 PUT
 	parseCommand "PUT-FROM" = Proto.parse1 PUT_FROM

From 27c8a4a229312fda465c3e82850c4920bff11800 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 17 Nov 2016 21:56:02 -0400
Subject: [PATCH 055/367] add CHECKPRESENT

Using SUCCESS to mean the content is present and FAILURE to mean it's not.
---
 Remote/Helper/P2P.hs | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/Remote/Helper/P2P.hs b/Remote/Helper/P2P.hs
index 19647e5d90..666dc84bee 100644
--- a/Remote/Helper/P2P.hs
+++ b/Remote/Helper/P2P.hs
@@ -13,6 +13,7 @@ module Remote.Helper.P2P (
 	runPure,
 	protoDump,
 	auth,
+	checkPresent,
 	remove,
 	get,
 	put,
@@ -45,6 +46,7 @@ data Message
 	= AUTH UUID AuthToken -- uuid of the peer that is authenticating
 	| AUTH_SUCCESS UUID -- uuid of the remote peer
 	| AUTH_FAILURE
+	| CHECKPRESENT Key
 	| REMOVE Key
 	| GET Offset Key
 	| PUT Key
@@ -77,7 +79,7 @@ data ProtoF next
 	-- content been transferred.
 	| CheckAuthToken UUID AuthToken (Bool -> next)
 	| SetPresent Key UUID next
-	| CheckPresent Key (Bool -> next)
+	| CheckContentPresent Key (Bool -> next)
 	-- ^ Checks if the whole content of the key is locally present.
 	| RemoveKeyFile Key (Bool -> next)
 	-- ^ If the key file is not present, still succeeds.
@@ -101,8 +103,8 @@ runPure (Free (ReadKeyFile _ _ next)) ms = runPure (next L.empty) ms
 runPure (Free (WriteKeyFile _ _ _ _ next)) ms = runPure (next True) ms
 runPure (Free (CheckAuthToken _ _ next)) ms = runPure (next True) ms
 runPure (Free (SetPresent _ _ next)) ms = runPure next ms
-runPure (Free (CheckPresent _ next)) ms = runPure (next False) ms
-runPure (Free (RemoveKeyFile _ next)) ms = runPure (next False) ms
+runPure (Free (CheckContentPresent _ next)) ms = runPure (next False) ms
+runPure (Free (RemoveKeyFile _ next)) ms = runPure (next True) ms
 
 protoDump :: [(String, Maybe Message)] -> String
 protoDump = unlines . map protoDump'
@@ -122,6 +124,11 @@ auth myuuid t = do
 			sendMessage (PROTO_ERROR "auth failed")
 			return Nothing
 
+checkPresent :: Key -> Proto Bool
+checkPresent key = do
+	sendMessage (CHECKPRESENT key)
+	checkSuccess
+
 remove :: Key -> Proto Bool
 remove key = do
 	sendMessage (REMOVE key)
@@ -176,11 +183,14 @@ serve myuuid = go Nothing
 				go autheduuid
 	
 	authed _theiruuid r = case r of
+		CHECKPRESENT key -> do
+			ok <- checkContentPresent key
+			sendMessage $ if ok then SUCCESS else FAILURE
 		REMOVE key -> do
 			ok <- removeKeyFile key
 			sendMessage $ if ok then SUCCESS else FAILURE
 		PUT key -> do
-			have <- checkPresent key
+			have <- checkContentPresent key
 			if have
 				then sendMessage ALREADY_HAVE
 				else do
@@ -241,6 +251,7 @@ instance Proto.Sendable Message where
 	formatMessage (AUTH uuid authtoken) = ["AUTH", Proto.serialize uuid, Proto.serialize authtoken]
 	formatMessage (AUTH_SUCCESS uuid) = ["AUTH-SUCCESS",  Proto.serialize uuid]
 	formatMessage AUTH_FAILURE = ["AUTH-FAILURE"]
+	formatMessage (CHECKPRESENT key) = ["CHECKPRESENT", Proto.serialize key]
 	formatMessage (REMOVE key) = ["REMOVE", Proto.serialize key]
 	formatMessage (GET offset key) = ["GET", Proto.serialize offset, Proto.serialize key]
 	formatMessage (PUT key) = ["PUT", Proto.serialize key]
@@ -255,6 +266,7 @@ instance Proto.Receivable Message where
 	parseCommand "AUTH" = Proto.parse2 AUTH
 	parseCommand "AUTH-SUCCESS" = Proto.parse1 AUTH_SUCCESS
 	parseCommand "AUTH-FAILURE" = Proto.parse0 AUTH_FAILURE
+	parseCommand "CHECKPRESENT" = Proto.parse1 CHECKPRESENT
 	parseCommand "REMOVE" = Proto.parse1 REMOVE
 	parseCommand "GET" = Proto.parse2 GET
 	parseCommand "PUT" = Proto.parse1 PUT

From b121078b3531606d582ae2eafcce98e29aba1bdc Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 17 Nov 2016 22:06:59 -0400
Subject: [PATCH 056/367] refactor

---
 Remote/Helper/P2P.hs | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/Remote/Helper/P2P.hs b/Remote/Helper/P2P.hs
index 666dc84bee..52f861ac99 100644
--- a/Remote/Helper/P2P.hs
+++ b/Remote/Helper/P2P.hs
@@ -55,7 +55,7 @@ data Message
 	| SUCCESS
 	| FAILURE
 	| DATA Len -- followed by bytes
-	| PROTO_ERROR String
+	| ERROR String
 	deriving (Show)
 
 -- | Free monad for implementing actions that use the protocol.
@@ -121,7 +121,7 @@ auth myuuid t = do
 		AUTH_SUCCESS theiruuid -> return $ Just theiruuid
 		AUTH_FAILURE -> return Nothing
 		_ -> do
-			sendMessage (PROTO_ERROR "auth failed")
+			sendMessage (ERROR "auth failed")
 			return Nothing
 
 checkPresent :: Key -> Proto Bool
@@ -145,7 +145,7 @@ put key = do
 		PUT_FROM offset -> sendContent key offset
 		ALREADY_HAVE -> return True
 		_ -> do
-			sendMessage (PROTO_ERROR "expected PUT_FROM")
+			sendMessage (ERROR "expected PUT_FROM")
 			return False
 
 -- | Serve the protocol.
@@ -157,7 +157,7 @@ put key = do
 -- talking to a server that does not support some new feature, and fall
 -- back.
 --
--- When the client sends PROTO_ERROR to the server, the server gives up,
+-- When the client sends ERROR to the server, the server gives up,
 -- since it's not clear what state the client is is, and so not possible to
 -- recover.
 serve :: UUID -> Proto ()
@@ -175,20 +175,16 @@ serve myuuid = go Nothing
 					else do
 						sendMessage AUTH_FAILURE
 						go autheduuid
-			PROTO_ERROR _ -> return ()
+			ERROR _ -> return ()
 			_ -> do
 				case autheduuid of
 					Just theiruuid -> authed theiruuid r
-					Nothing -> sendMessage (PROTO_ERROR "must AUTH first")
+					Nothing -> sendMessage (ERROR "must AUTH first")
 				go autheduuid
 	
 	authed _theiruuid r = case r of
-		CHECKPRESENT key -> do
-			ok <- checkContentPresent key
-			sendMessage $ if ok then SUCCESS else FAILURE
-		REMOVE key -> do
-			ok <- removeKeyFile key
-			sendMessage $ if ok then SUCCESS else FAILURE
+		CHECKPRESENT key -> sendSuccess =<< checkContentPresent key
+		REMOVE key -> sendSuccess =<< removeKeyFile key
 		PUT key -> do
 			have <- checkContentPresent key
 			if have
@@ -200,7 +196,7 @@ serve myuuid = go Nothing
 		-- setPresent not called because the peer may have
 		-- requested the data but not permanatly stored it.
 		GET offset key -> void $ sendContent key offset
-		_ -> sendMessage (PROTO_ERROR "unexpected command")
+		_ -> sendMessage (ERROR "unexpected command")
 
 sendContent :: Key -> Offset -> Proto Bool
 sendContent key offset = do
@@ -218,10 +214,10 @@ receiveContent key mkmsg = do
 	case r of
 		DATA len -> do
 			ok <- writeKeyFile key offset len =<< receiveBytes len
-			sendMessage $ if ok then SUCCESS else FAILURE
+			sendSuccess ok
 			return ok
 		_ -> do
-			sendMessage (PROTO_ERROR "expected DATA")
+			sendMessage (ERROR "expected DATA")
 			return False
 
 checkSuccess :: Proto Bool
@@ -231,9 +227,13 @@ checkSuccess = do
 		SUCCESS -> return True
 		FAILURE -> return False
 		_ -> do
-			sendMessage (PROTO_ERROR "expected SUCCESS or FAILURE")
+			sendMessage (ERROR "expected SUCCESS or FAILURE")
 			return False
 
+sendSuccess :: Bool -> Proto ()
+sendSuccess True = sendMessage SUCCESS
+sendSuccess False = sendMessage FAILURE
+
 -- Reads key file from an offset. The Len should correspond to
 -- the length of the ByteString, but to avoid buffering the content
 -- in memory, is gotten using keyFileSize.
@@ -260,7 +260,7 @@ instance Proto.Sendable Message where
 	formatMessage SUCCESS = ["SUCCESS"]
 	formatMessage FAILURE = ["FAILURE"]
 	formatMessage (DATA leng) = ["DATA", Proto.serialize leng]
-	formatMessage (PROTO_ERROR err) = ["PROTO-ERROR", Proto.serialize err]
+	formatMessage (ERROR err) = ["ERROR", Proto.serialize err]
 
 instance Proto.Receivable Message where
 	parseCommand "AUTH" = Proto.parse2 AUTH
@@ -275,7 +275,7 @@ instance Proto.Receivable Message where
 	parseCommand "SUCCESS" = Proto.parse0 SUCCESS
 	parseCommand "FAILURE" = Proto.parse0 FAILURE
 	parseCommand "DATA" = Proto.parse1 DATA
-	parseCommand "PROTO-ERROR" = Proto.parse1 PROTO_ERROR
+	parseCommand "ERROR" = Proto.parse1 ERROR
 	parseCommand _ = Proto.parseFail
 
 instance Proto.Serializable Offset where

From 236ff111a7f32a38fca5dce7eaabe5860be7302e Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 17 Nov 2016 22:10:28 -0400
Subject: [PATCH 057/367] rename

---
 Remote/Helper/P2P.hs | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/Remote/Helper/P2P.hs b/Remote/Helper/P2P.hs
index 52f861ac99..38859914a0 100644
--- a/Remote/Helper/P2P.hs
+++ b/Remote/Helper/P2P.hs
@@ -61,7 +61,7 @@ data Message
 -- | Free monad for implementing actions that use the protocol.
 data ProtoF next
 	= SendMessage Message next
-	| GetMessage (Message -> next)
+	| ReceiveMessage (Message -> next)
 	| SendBytes Len L.ByteString next
 	| ReceiveBytes Len (L.ByteString -> next)
 	-- ^ Lazily reads bytes from peer. Stops once Len are read,
@@ -94,8 +94,8 @@ $(makeFree ''ProtoF)
 runPure :: Show r => Proto r -> [Message] -> [(String, Maybe Message)]
 runPure (Pure r) _ = [("result: " ++ show r, Nothing)]
 runPure (Free (SendMessage m next)) ms = (">",  Just m):runPure next ms
-runPure (Free (GetMessage _)) [] = [("not enough Messages provided", Nothing)]
-runPure (Free (GetMessage next)) (m:ms) = ("<", Just m):runPure (next m) ms
+runPure (Free (ReceiveMessage _)) [] = [("not enough Messages provided", Nothing)]
+runPure (Free (ReceiveMessage next)) (m:ms) = ("<", Just m):runPure (next m) ms
 runPure (Free (SendBytes _ _ next)) ms = ("> bytes", Nothing):runPure next ms
 runPure (Free (ReceiveBytes _ next)) ms = ("< bytes", Nothing):runPure (next L.empty) ms
 runPure (Free (KeyFileSize _ next)) ms = runPure (next (Len 100)) ms
@@ -116,7 +116,7 @@ protoDump' (s, Just m) = s ++ " " ++ unwords (Proto.formatMessage m)
 auth :: UUID -> AuthToken -> Proto (Maybe UUID)
 auth myuuid t = do
 	sendMessage (AUTH myuuid t)
-	r <- getMessage
+	r <- receiveMessage
 	case r of
 		AUTH_SUCCESS theiruuid -> return $ Just theiruuid
 		AUTH_FAILURE -> return Nothing
@@ -140,7 +140,7 @@ get key = receiveContent key (`GET` key)
 put :: Key -> Proto Bool
 put key = do
 	sendMessage (PUT key)
-	r <- getMessage
+	r <- receiveMessage
 	case r of
 		PUT_FROM offset -> sendContent key offset
 		ALREADY_HAVE -> return True
@@ -164,7 +164,7 @@ serve :: UUID -> Proto ()
 serve myuuid = go Nothing
   where
 	go autheduuid = do
-		r <- getMessage
+		r <- receiveMessage
 		case r of
 			AUTH theiruuid authtoken -> do
 				ok <- checkAuthToken theiruuid authtoken
@@ -210,7 +210,7 @@ receiveContent key mkmsg = do
 	Len n <- keyFileSize key
 	let offset = Offset n
 	sendMessage (mkmsg offset)
-	r <- getMessage
+	r <- receiveMessage
 	case r of
 		DATA len -> do
 			ok <- writeKeyFile key offset len =<< receiveBytes len
@@ -222,7 +222,7 @@ receiveContent key mkmsg = do
 
 checkSuccess :: Proto Bool
 checkSuccess = do
-	ack <- getMessage
+	ack <- receiveMessage
 	case ack of
 		SUCCESS -> return True
 		FAILURE -> return False

From 73a6b9b51455f2ae8483a86a98e9863fffe9ebac Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 18 Nov 2016 01:32:24 -0400
Subject: [PATCH 058/367] Add content locking to P2P protocol

Is content locking needed in the P2P protocol? Based on re-reading
bugs/concurrent_drop--from_presence_checking_failures.mdwn,
I think so: Peers can form cycles, and multiple peers can all be trying
to drop the same content.

So, added content locking to the protocol, with some difficulty.

The implementation is fine as far as it goes, but note the warning
comment for lockContentWhile -- if the connection to the peer is dropped
unexpectedly, the peer will then unlock the content, and yet the local
side will still think it's locked.

To be honest I'm not sure if Remote.Git's lockKey for ssh remotes
doesn't have the same problem. It checks that the
"ssh remote git-annex-shell lockcontent"
process has not exited, but if the connection closes afer that check,
the lockcontent command will unlock it, and yet the local side will
still think it's locked.

Probably this needs to be fixed by eg, making lockcontent catch any
execptions due to the connection closing, and in that case, wait a
significantly long time before dropping the lock.

This commit was sponsored by Anthony DeRobertis on Patreon.
---
 Remote/Helper/P2P.hs | 41 ++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 40 insertions(+), 1 deletion(-)

diff --git a/Remote/Helper/P2P.hs b/Remote/Helper/P2P.hs
index 38859914a0..a62f7c03d5 100644
--- a/Remote/Helper/P2P.hs
+++ b/Remote/Helper/P2P.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-{-# LANGUAGE DeriveFunctor, TemplateHaskell, FlexibleContexts #-}
+{-# LANGUAGE DeriveFunctor, TemplateHaskell, FlexibleContexts, RankNTypes #-}
 
 module Remote.Helper.P2P (
 	AuthToken(..),
@@ -14,6 +14,7 @@ module Remote.Helper.P2P (
 	protoDump,
 	auth,
 	checkPresent,
+	lockContentWhile,
 	remove,
 	get,
 	put,
@@ -29,6 +30,7 @@ import Utility.PartialPrelude
 import Control.Monad
 import Control.Monad.Free
 import Control.Monad.Free.TH
+import Control.Monad.Catch
 import qualified Data.ByteString.Lazy as L
 
 newtype AuthToken = AuthToken String
@@ -47,6 +49,8 @@ data Message
 	| AUTH_SUCCESS UUID -- uuid of the remote peer
 	| AUTH_FAILURE
 	| CHECKPRESENT Key
+	| LOCKCONTENT Key
+	| UNLOCKCONTENT
 	| REMOVE Key
 	| GET Offset Key
 	| PUT Key
@@ -84,6 +88,9 @@ data ProtoF next
 	| RemoveKeyFile Key (Bool -> next)
 	-- ^ If the key file is not present, still succeeds.
 	-- May fail if not enough copies to safely drop, etc.
+	| TryLockContent Key (Bool -> Proto ()) next
+	-- ^ Try to lock the content of a key,  preventing it
+	-- from being deleted, and run the provided protocol action.
 	deriving (Functor)
 
 type Proto = Free ProtoF
@@ -105,6 +112,7 @@ runPure (Free (CheckAuthToken _ _ next)) ms = runPure (next True) ms
 runPure (Free (SetPresent _ _ next)) ms = runPure next ms
 runPure (Free (CheckContentPresent _ next)) ms = runPure (next False) ms
 runPure (Free (RemoveKeyFile _ next)) ms = runPure (next True) ms
+runPure (Free (TryLockContent _ p next)) ms = runPure (p True >> next) ms
 
 protoDump :: [(String, Maybe Message)] -> String
 protoDump = unlines . map protoDump'
@@ -129,6 +137,26 @@ checkPresent key = do
 	sendMessage (CHECKPRESENT key)
 	checkSuccess
 
+{- Locks content to prevent it from being dropped, while running an action.
+ -
+ - Note that this only guarantees that the content is locked as long as the
+ - connection to the peer remains up. If the connection is unexpectededly
+ - dropped, the peer will then unlock the content.
+ -}
+lockContentWhile 
+	:: MonadMask m 
+	=> (forall r. Proto r -> m r)
+	-> Key
+	-> (Bool -> m ())
+	-> m ()
+lockContentWhile runproto key a = bracket setup cleanup a
+  where
+	setup = runproto $ do
+		sendMessage (LOCKCONTENT key)
+		checkSuccess
+	cleanup True = runproto $ sendMessage UNLOCKCONTENT
+	cleanup False = return ()
+
 remove :: Key -> Proto Bool
 remove key = do
 	sendMessage (REMOVE key)
@@ -183,6 +211,13 @@ serve myuuid = go Nothing
 				go autheduuid
 	
 	authed _theiruuid r = case r of
+		LOCKCONTENT key -> tryLockContent key $ \locked -> do
+			sendSuccess locked
+			when locked $ do
+				r' <- receiveMessage
+				case r' of
+					UNLOCKCONTENT -> return ()
+					_ -> sendMessage (ERROR "expected UNLOCKCONTENT")
 		CHECKPRESENT key -> sendSuccess =<< checkContentPresent key
 		REMOVE key -> sendSuccess =<< removeKeyFile key
 		PUT key -> do
@@ -252,6 +287,8 @@ instance Proto.Sendable Message where
 	formatMessage (AUTH_SUCCESS uuid) = ["AUTH-SUCCESS",  Proto.serialize uuid]
 	formatMessage AUTH_FAILURE = ["AUTH-FAILURE"]
 	formatMessage (CHECKPRESENT key) = ["CHECKPRESENT", Proto.serialize key]
+	formatMessage (LOCKCONTENT key) = ["LOCKCONTENT", Proto.serialize key]
+	formatMessage UNLOCKCONTENT = ["UNLOCKCONTENT"]
 	formatMessage (REMOVE key) = ["REMOVE", Proto.serialize key]
 	formatMessage (GET offset key) = ["GET", Proto.serialize offset, Proto.serialize key]
 	formatMessage (PUT key) = ["PUT", Proto.serialize key]
@@ -267,6 +304,8 @@ instance Proto.Receivable Message where
 	parseCommand "AUTH-SUCCESS" = Proto.parse1 AUTH_SUCCESS
 	parseCommand "AUTH-FAILURE" = Proto.parse0 AUTH_FAILURE
 	parseCommand "CHECKPRESENT" = Proto.parse1 CHECKPRESENT
+	parseCommand "LOCKCONTENT" = Proto.parse1 LOCKCONTENT
+	parseCommand "UNLOCKCONTENT" = Proto.parse0 UNLOCKCONTENT
 	parseCommand "REMOVE" = Proto.parse1 REMOVE
 	parseCommand "GET" = Proto.parse2 GET
 	parseCommand "PUT" = Proto.parse1 PUT

From 4baae17fa40a4ca01ba204a0d21cde92ba445199 Mon Sep 17 00:00:00 2001
From: openmedi <openmedi@web>
Date: Fri, 18 Nov 2016 12:03:44 +0000
Subject: [PATCH 059/367] Added a comment

---
 .../comment_12_f2e570dc60a6f16e8f696d94e253775f._comment   | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 doc/forum/how_to_disaster_recovery/comment_12_f2e570dc60a6f16e8f696d94e253775f._comment

diff --git a/doc/forum/how_to_disaster_recovery/comment_12_f2e570dc60a6f16e8f696d94e253775f._comment b/doc/forum/how_to_disaster_recovery/comment_12_f2e570dc60a6f16e8f696d94e253775f._comment
new file mode 100644
index 0000000000..7c7da0ef47
--- /dev/null
+++ b/doc/forum/how_to_disaster_recovery/comment_12_f2e570dc60a6f16e8f696d94e253775f._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="openmedi"
+ subject="comment 12"
+ date="2016-11-18T12:03:44Z"
+ content="""
+A recent update to annex via homebrew now reslolves the issue with the weird looking webapp.
+"""]]

From fe556ced50cd56009a3920506d6918c3bcab15d0 Mon Sep 17 00:00:00 2001
From: openmedi <openmedi@web>
Date: Fri, 18 Nov 2016 13:57:49 +0000
Subject: [PATCH 060/367]

---
 ...hat_to_do_if_special_remotes_refuses_drops__63__.mdwn | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/forum/What_to_do_if_special_remotes_refuses_drops__63__.mdwn

diff --git a/doc/forum/What_to_do_if_special_remotes_refuses_drops__63__.mdwn b/doc/forum/What_to_do_if_special_remotes_refuses_drops__63__.mdwn
new file mode 100644
index 0000000000..512e89528c
--- /dev/null
+++ b/doc/forum/What_to_do_if_special_remotes_refuses_drops__63__.mdwn
@@ -0,0 +1,9 @@
+I have a special remote that I would like to delete and have marked it as such in the assistant. Although this was before my myriad of problems with git annex itself wanting to repair the repo all the time. Right now if I take a loog into my daemon.log I see the following error over and over again:
+
+```
+drop skydrive foo.bar 
+  This file could not be removed
+failed
+```
+
+I checked if I can login into my account and it works just fine. So I assume that this might be a bug? Is it somehow possible to forego the cleaning out of the special remote and just mark it as deleted for good? Thanks in advance!

From 0a34f08ad982d2a7a40dd01ce8e85c2ff45e2e30 Mon Sep 17 00:00:00 2001
From: yomguy <yomguy@web>
Date: Fri, 18 Nov 2016 14:00:51 +0000
Subject: [PATCH 061/367] Added a comment

---
 ..._a681a4847acbe890c4e486288b3c81d3._comment | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 doc/forum/uuid_mismatch___59___expected_Just___40__UUID_...___41___but_remote_gitrepo_has_UUID_.../comment_3_a681a4847acbe890c4e486288b3c81d3._comment

diff --git a/doc/forum/uuid_mismatch___59___expected_Just___40__UUID_...___41___but_remote_gitrepo_has_UUID_.../comment_3_a681a4847acbe890c4e486288b3c81d3._comment b/doc/forum/uuid_mismatch___59___expected_Just___40__UUID_...___41___but_remote_gitrepo_has_UUID_.../comment_3_a681a4847acbe890c4e486288b3c81d3._comment
new file mode 100644
index 0000000000..d92f3fe17a
--- /dev/null
+++ b/doc/forum/uuid_mismatch___59___expected_Just___40__UUID_...___41___but_remote_gitrepo_has_UUID_.../comment_3_a681a4847acbe890c4e486288b3c81d3._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="yomguy"
+ avatar="http://cdn.libravatar.org/avatar/03db077c04f8b753f3f504d9a2b06a29"
+ subject="comment 3"
+ date="2016-11-18T14:00:51Z"
+ content="""
+Hi joey,
+
+After modifying the gcrypt-id as you proposed, I have finally managed to clone the repo with  
+
+`git clone gcrypt::ssh://my.domain/home/admin/`
+
+But now I get only unresolved symbolic links for each files, that is .git/annex/objects directory only contains .map files.
+
+Would you have an idea about the reason/source of this behavior?
+
+Thank you so much,
+Guillaume
+"""]]

From 50ad13b809fe1fc7be5dfa0ecff938554df2c5cc Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 18 Nov 2016 11:40:52 -0400
Subject: [PATCH 062/367] comment

---
 ...nt_1_0b523b2b6c361346c36ad456bbbac645._comment | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/forum/What_to_do_if_special_remotes_refuses_drops__63__/comment_1_0b523b2b6c361346c36ad456bbbac645._comment

diff --git a/doc/forum/What_to_do_if_special_remotes_refuses_drops__63__/comment_1_0b523b2b6c361346c36ad456bbbac645._comment b/doc/forum/What_to_do_if_special_remotes_refuses_drops__63__/comment_1_0b523b2b6c361346c36ad456bbbac645._comment
new file mode 100644
index 0000000000..cf808aac4b
--- /dev/null
+++ b/doc/forum/What_to_do_if_special_remotes_refuses_drops__63__/comment_1_0b523b2b6c361346c36ad456bbbac645._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-11-18T15:39:20Z"
+ content="""
+It could certianly be a bug in the special remote implementation. It's also
+possible for some special remotes to intentionally not be able to remove
+content (this is the case with the web special remote, and the bup special
+remote at least).
+
+You can manually remove the special remote, by editing .git/config and
+deleting the stanza for that remote. You may want to run `git annex dead
+$remotename` first, if you don't intend to ever use that special remote
+again.
+"""]]

From 5680565122577a1dbe68ac0d541b83301306f6d2 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 18 Nov 2016 11:59:49 -0400
Subject: [PATCH 063/367] releasing package git-annex version 6.20161118

---
 CHANGELOG       | 4 ++--
 git-annex.cabal | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index 5bd9d3e9d7..3777e6d5ad 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,4 +1,4 @@
-git-annex (6.20161112) UNRELEASED; urgency=medium
+git-annex (6.20161118) unstable; urgency=medium
 
   * git-annex.cabal: Loosen bounds on persistent to allow 2.5, which
     on Debian has been patched to work with esqueleto.
@@ -16,7 +16,7 @@ git-annex (6.20161112) UNRELEASED; urgency=medium
   * Linux arm standalone: Build with a 32kb page size, which is needed
     on several ARM NAS devices, including Drobo 5N, and WD NAS.
 
- -- Joey Hess <id@joeyh.name>  Tue, 15 Nov 2016 11:15:27 -0400
+ -- Joey Hess <id@joeyh.name>  Fri, 18 Nov 2016 11:43:14 -0400
 
 git-annex (6.20161111) unstable; urgency=medium
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 46b08d22d0..7535c50377 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 6.20161111
+Version: 6.20161118
 Cabal-Version: >= 1.8
 License: GPL-3
 Maintainer: Joey Hess <id@joeyh.name>

From c45ec7b81994a2394c6d168c3271abd9e88f34b9 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 18 Nov 2016 12:00:20 -0400
Subject: [PATCH 064/367] add news item for git-annex 6.20161118

---
 doc/news/version_6.20161118.mdwn | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 doc/news/version_6.20161118.mdwn

diff --git a/doc/news/version_6.20161118.mdwn b/doc/news/version_6.20161118.mdwn
new file mode 100644
index 0000000000..42d86282c1
--- /dev/null
+++ b/doc/news/version_6.20161118.mdwn
@@ -0,0 +1,17 @@
+git-annex 6.20161118 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * git-annex.cabal: Loosen bounds on persistent to allow 2.5, which
+     on Debian has been patched to work with esqueleto.
+     This may break cabal's resolver on non-Debian systems;
+     if so, either use stack to build, or run cabal with
+     --constraint='persistent ==2.2.4.1'
+     Hopefully this mess with esqueleto will be resolved soon.
+   * sync: Pass --allow-unrelated-histories to git merge when used with git
+     git 2.9.0 or newer. This makes merging a remote into a freshly created
+     direct mode repository work the same as it works in indirect mode.
+   * Avoid backtraces on expected failures when built with ghc 8;
+     only use backtraces for unexpected errors.
+   * fsck --all --from was checking the existence and content of files
+     in the local repository, rather than on the special remote. Oops.
+   * Linux arm standalone: Build with a 32kb page size, which is needed
+     on several ARM NAS devices, including Drobo 5N, and WD NAS."""]]
\ No newline at end of file

From ddfb3206642285e2994899eb1f5053d0419afa24 Mon Sep 17 00:00:00 2001
From: "t.z.mates" <t.z.mates@web>
Date: Sat, 19 Nov 2016 04:42:25 +0000
Subject: [PATCH 065/367] Added a comment

---
 ...mment_2_b3998823aca4266089dcbcf325d8f8c1._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_2_b3998823aca4266089dcbcf325d8f8c1._comment

diff --git a/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_2_b3998823aca4266089dcbcf325d8f8c1._comment b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_2_b3998823aca4266089dcbcf325d8f8c1._comment
new file mode 100644
index 0000000000..1046fb0665
--- /dev/null
+++ b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_2_b3998823aca4266089dcbcf325d8f8c1._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="t.z.mates"
+ avatar="http://cdn.libravatar.org/avatar/90f15fad216078fd08d62cc676487925"
+ subject="comment 2"
+ date="2016-11-19T04:42:25Z"
+ content="""
+Thanks for looking into it; I just checked again, and even on the newest version (6.20161118 binary), I'm still experiencing the behavior. However, I checked on an older OpenSuse box I have, and there it works (6.20161031 from OpenSuse repo).
+
+Since my two machines experiencing the problem are both running arch, it seems it's somehow related to that distro. I've checked both installing via the binary (from kitenet) and from the arch community repo, but both produce the same behavior. Further, the OpenSuse install has the same build flags as the binaries, so that doesn't seem to be it. Are there any other diagnostics I can run?
+
+This particular problem isn't very troublesome (it doesn't seem to have any material impact aside from error messages); however, I also occasionally experience a more serious bug. Namely, when certain (seemingly random) files are added to the repo locked, their content disappears and the symlink is broken (this is the other problem I alluded to in the description). I suspect that problem is related to this one though, since it also only affects my arch machines. I haven't yet submitted a report for that bug yet, though, since I can't reliably replicate it. 
+"""]]

From fc304a8ab91277024f9b0241dadadd35e6889e1b Mon Sep 17 00:00:00 2001
From: ilovezfs <ilovezfs@web>
Date: Sat, 19 Nov 2016 16:00:50 +0000
Subject: [PATCH 066/367]

---
 doc/bugs/Build_with_aws_head_fails.mdwn | 48 +++++++++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100644 doc/bugs/Build_with_aws_head_fails.mdwn

diff --git a/doc/bugs/Build_with_aws_head_fails.mdwn b/doc/bugs/Build_with_aws_head_fails.mdwn
new file mode 100644
index 0000000000..9937d6c35b
--- /dev/null
+++ b/doc/bugs/Build_with_aws_head_fails.mdwn
@@ -0,0 +1,48 @@
+### Please describe the problem.
+https://github.com/aristidb/aws/issues/206 was recently resolved in https://github.com/aristidb/aws/pull/213.
+
+A newer version will be tagged imminently according to https://github.com/aristidb/aws/issues/206#issuecomment-260214736.
+
+With the http-conduit (<2.2.0) constraint removed from git-annex.cabal, and the aws dependency set to use aws head (currently c8806dc), the git-annex build fails. 
+
+### What steps will reproduce the problem?
+
+Remove the http-conduit (<2.2.0) constraint and attempt to build git-annex with aws head.
+
+### What version of git-annex are you using? On what operating system?
+
+macOS 10.11, git-annex 6.20161118.
+
+### Please provide any additional information below.
+Full build log: https://gist.github.com/ilovezfs/15bcd8f1086b3d825beff58140e04eec
+[[!format sh """
+[ 90 of 542] Compiling Types.Crypto     ( Types/Crypto.hs, dist/dist-sandbox-6b15e8f0/build/git-annex/git-annex-tmp/Types/Crypto.o )
+[ 91 of 542] Compiling Utility.Metered  ( Utility/Metered.hs, dist/dist-sandbox-6b15e8f0/build/git-annex/git-annex-tmp/Utility/Metered.o )
+[ 92 of 542] Compiling Messages.JSON    ( Messages/JSON.hs, dist/dist-sandbox-6b15e8f0/build/git-annex/git-annex-tmp/Messages/JSON.o )
+[ 93 of 542] Compiling Utility.Url      ( Utility/Url.hs, dist/dist-sandbox-6b15e8f0/build/git-annex/git-annex-tmp/Utility/Url.o )
+
+Utility/Url.hs:354:34: error:
+    • The constructor ‘StatusCodeException’ should have 2 arguments, but has been given 3
+    • In the pattern: StatusCodeException s _ _
+      In an equation for ‘matchStatusCodeException’:
+          matchStatusCodeException want e@(StatusCodeException s _ _)
+            | want s = Just e
+            | otherwise = Nothing
+
+Utility/Url.hs:354:34: error:
+    • Couldn't match expected type ‘HttpException’
+                  with actual type ‘HttpExceptionContent’
+    • In the pattern: StatusCodeException s _ _
+      In an equation for ‘matchStatusCodeException’:
+          matchStatusCodeException want e@(StatusCodeException s _ _)
+            | want s = Just e
+            | otherwise = Nothing
+cabal: Leaving directory '.'
+cabal: Error: some packages failed to install:
+git-annex-6.20161118 failed during the building phase. The exception was:
+ExitFailure 1
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+Yes :)
+

From 0eaad7ca3a847b0f56ed2dbf72d06a33f41749c8 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 19 Nov 2016 16:30:57 -0400
Subject: [PATCH 067/367] extend p2p protocol to support gitremote-helpers
 connect

A bit tricky since Proto doesn't support threads. Rather than adding
threading support to it, ended up using a callback that waits for both
data on a Handle, and incoming messages at the same time.

This commit was sponsored by Denis Dzyubenko on Patreon.
---
 Remote/Helper/P2P.hs      | 72 ++++++++++++++++++++++++++++++++++++---
 Utility/SimpleProtocol.hs |  7 ++++
 2 files changed, 74 insertions(+), 5 deletions(-)

diff --git a/Remote/Helper/P2P.hs b/Remote/Helper/P2P.hs
index a62f7c03d5..d3d3dfa088 100644
--- a/Remote/Helper/P2P.hs
+++ b/Remote/Helper/P2P.hs
@@ -18,6 +18,7 @@ module Remote.Helper.P2P (
 	remove,
 	get,
 	put,
+	connect,
 	serve,
 ) where
 
@@ -31,6 +32,8 @@ import Control.Monad
 import Control.Monad.Free
 import Control.Monad.Free.TH
 import Control.Monad.Catch
+import System.Exit (ExitCode(..))
+import System.IO (Handle)
 import qualified Data.ByteString.Lazy as L
 
 newtype AuthToken = AuthToken String
@@ -42,12 +45,22 @@ newtype Offset = Offset Integer
 newtype Len = Len Integer
 	deriving (Show)
 
+-- | Service as used by the connect message is gitremote-helpers(1)
+data Service = UploadPack | ReceivePack
+	deriving (Show)
+
+data RelayData
+	= RelayData L.ByteString
+	| RelayMessage Message
+
 -- | Messages in the protocol. The peer that makes the connection
 -- always initiates requests, and the other peer makes responses to them.
 data Message
 	= AUTH UUID AuthToken -- uuid of the peer that is authenticating
 	| AUTH_SUCCESS UUID -- uuid of the remote peer
 	| AUTH_FAILURE
+	| CONNECT Service
+	| CONNECTDONE ExitCode
 	| CHECKPRESENT Key
 	| LOCKCONTENT Key
 	| UNLOCKCONTENT
@@ -58,7 +71,7 @@ data Message
 	| ALREADY_HAVE
 	| SUCCESS
 	| FAILURE
-	| DATA Len -- followed by bytes
+	| DATA Len -- followed by bytes of data
 	| ERROR String
 	deriving (Show)
 
@@ -89,8 +102,20 @@ data ProtoF next
 	-- ^ If the key file is not present, still succeeds.
 	-- May fail if not enough copies to safely drop, etc.
 	| TryLockContent Key (Bool -> Proto ()) next
+	| WriteHandle Handle L.ByteString next
 	-- ^ Try to lock the content of a key,  preventing it
 	-- from being deleted, and run the provided protocol action.
+	| Relay Handle (RelayData -> Proto (Maybe ExitCode)) (ExitCode -> next)
+	-- ^ Waits for data to be written to the Handle, and for messages
+	-- to be received from the peer, and passes the data to the
+	-- callback, continuing until it returns an ExitCode.
+	| RelayService Service
+		(Handle -> RelayData -> Proto (Maybe ExitCode))
+		(ExitCode -> next)
+	-- ^ Runs a service, and waits for it to output to stdout,
+	-- and for messages to be received from the peer, and passes
+	-- the data to the callback (which is also passed the service's
+	-- stdin Handle), continuing uniil the service exits.
 	deriving (Functor)
 
 type Proto = Free ProtoF
@@ -113,6 +138,9 @@ runPure (Free (SetPresent _ _ next)) ms = runPure next ms
 runPure (Free (CheckContentPresent _ next)) ms = runPure (next False) ms
 runPure (Free (RemoveKeyFile _ next)) ms = runPure (next True) ms
 runPure (Free (TryLockContent _ p next)) ms = runPure (p True >> next) ms
+runPure (Free (WriteHandle _ _ next)) ms = runPure next ms
+runPure (Free (Relay _ _ next)) ms = runPure (next ExitSuccess) ms
+runPure (Free (RelayService _ _ next)) ms = runPure (next ExitSuccess) ms
 
 protoDump :: [(String, Maybe Message)] -> String
 protoDump = unlines . map protoDump'
@@ -176,6 +204,26 @@ put key = do
 			sendMessage (ERROR "expected PUT_FROM")
 			return False
 
+connect :: Service -> Handle -> Handle -> Proto ExitCode
+connect service hin hout = do
+	sendMessage (CONNECT service)
+	relay hin (relayCallback hout)
+
+relayCallback :: Handle -> RelayData -> Proto (Maybe ExitCode)
+relayCallback hout (RelayMessage (DATA len)) = do
+	writeHandle hout =<< receiveBytes len
+	return Nothing
+relayCallback _ (RelayMessage (CONNECTDONE exitcode)) =
+	return (Just exitcode)
+relayCallback _ (RelayMessage _) = do
+	sendMessage (ERROR "expected DATA or CONNECTDONE")
+	return (Just (ExitFailure 1))
+relayCallback _ (RelayData b) = do
+	let len = Len $ fromIntegral $ L.length b
+	sendMessage (DATA len)
+	sendBytes len b
+	return Nothing
+
 -- | Serve the protocol.
 --
 -- Note that if the client sends an unexpected message, the server will
@@ -231,11 +279,14 @@ serve myuuid = go Nothing
 		-- setPresent not called because the peer may have
 		-- requested the data but not permanatly stored it.
 		GET offset key -> void $ sendContent key offset
+		CONNECT service -> do
+			exitcode <- relayService service relayCallback
+			sendMessage (CONNECTDONE exitcode)
 		_ -> sendMessage (ERROR "unexpected command")
 
 sendContent :: Key -> Offset -> Proto Bool
 sendContent key offset = do
-	(len, content) <- readKeyFile' key offset
+	(len, content) <- readKeyFileLen key offset
 	sendMessage (DATA len)
 	sendBytes len content
 	checkSuccess
@@ -272,8 +323,8 @@ sendSuccess False = sendMessage FAILURE
 -- Reads key file from an offset. The Len should correspond to
 -- the length of the ByteString, but to avoid buffering the content
 -- in memory, is gotten using keyFileSize.
-readKeyFile' :: Key -> Offset -> Proto (Len, L.ByteString)
-readKeyFile' key (Offset offset) = do
+readKeyFileLen :: Key -> Offset -> Proto (Len, L.ByteString)
+readKeyFileLen key (Offset offset) = do
 	(Len totallen) <- keyFileSize key
 	let len = totallen - offset
 	if len <= 0
@@ -286,6 +337,8 @@ instance Proto.Sendable Message where
 	formatMessage (AUTH uuid authtoken) = ["AUTH", Proto.serialize uuid, Proto.serialize authtoken]
 	formatMessage (AUTH_SUCCESS uuid) = ["AUTH-SUCCESS",  Proto.serialize uuid]
 	formatMessage AUTH_FAILURE = ["AUTH-FAILURE"]
+	formatMessage (CONNECT service) = ["CONNECT", Proto.serialize service]
+	formatMessage (CONNECTDONE exitcode) = ["CONNECTDONE", Proto.serialize exitcode]
 	formatMessage (CHECKPRESENT key) = ["CHECKPRESENT", Proto.serialize key]
 	formatMessage (LOCKCONTENT key) = ["LOCKCONTENT", Proto.serialize key]
 	formatMessage UNLOCKCONTENT = ["UNLOCKCONTENT"]
@@ -296,13 +349,15 @@ instance Proto.Sendable Message where
 	formatMessage ALREADY_HAVE = ["ALREADY-HAVE"]
 	formatMessage SUCCESS = ["SUCCESS"]
 	formatMessage FAILURE = ["FAILURE"]
-	formatMessage (DATA leng) = ["DATA", Proto.serialize leng]
+	formatMessage (DATA len) = ["DATA", Proto.serialize len]
 	formatMessage (ERROR err) = ["ERROR", Proto.serialize err]
 
 instance Proto.Receivable Message where
 	parseCommand "AUTH" = Proto.parse2 AUTH
 	parseCommand "AUTH-SUCCESS" = Proto.parse1 AUTH_SUCCESS
 	parseCommand "AUTH-FAILURE" = Proto.parse0 AUTH_FAILURE
+	parseCommand "CONNECT" = Proto.parse1 CONNECT
+	parseCommand "CONNECTDONE" = Proto.parse1 CONNECT
 	parseCommand "CHECKPRESENT" = Proto.parse1 CHECKPRESENT
 	parseCommand "LOCKCONTENT" = Proto.parse1 LOCKCONTENT
 	parseCommand "UNLOCKCONTENT" = Proto.parse0 UNLOCKCONTENT
@@ -328,3 +383,10 @@ instance Proto.Serializable Len where
 instance Proto.Serializable AuthToken where
 	serialize (AuthToken s) = s
 	deserialize = Just . AuthToken
+
+instance Proto.Serializable Service where
+	serialize UploadPack = "git-upload-pack"
+	serialize ReceivePack = "git-receive-pack"
+	deserialize "git-upload-pack" = Just UploadPack
+	deserialize "git-receive-pack" = Just ReceivePack
+	deserialize _ = Nothing
diff --git a/Utility/SimpleProtocol.hs b/Utility/SimpleProtocol.hs
index 728b135e8d..473129218a 100644
--- a/Utility/SimpleProtocol.hs
+++ b/Utility/SimpleProtocol.hs
@@ -24,6 +24,7 @@ module Utility.SimpleProtocol (
 
 import Data.Char
 import GHC.IO.Handle
+import System.Exit (ExitCode(..))
 
 import Common
 
@@ -95,3 +96,9 @@ dupIoHandles = do
 instance Serializable [Char] where
 	serialize = id
 	deserialize = Just
+
+instance Serializable ExitCode where
+	serialize ExitSuccess = "0"
+	serialize (ExitFailure n) = show n
+	deserialize "0" = Just ExitSuccess
+	deserialize s = ExitFailure <$> readish s

From b0c190002c7b8b6c44d9e5f9144a613c842a9fa1 Mon Sep 17 00:00:00 2001
From: "justin.lebar@7a36fcafc322d9a381e89f08ab6289033c6dde91"
 <justinlebar@web>
Date: Sun, 20 Nov 2016 03:47:23 +0000
Subject: [PATCH 068/367] Added a comment

---
 ..._2_c227071f23a96ed9928f128e7f77e503._comment | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 doc/bugs/Nearline_bucket_stopped_working___40__can__39__t_even_HEAD_files__41__/comment_2_c227071f23a96ed9928f128e7f77e503._comment

diff --git a/doc/bugs/Nearline_bucket_stopped_working___40__can__39__t_even_HEAD_files__41__/comment_2_c227071f23a96ed9928f128e7f77e503._comment b/doc/bugs/Nearline_bucket_stopped_working___40__can__39__t_even_HEAD_files__41__/comment_2_c227071f23a96ed9928f128e7f77e503._comment
new file mode 100644
index 0000000000..820e6b0401
--- /dev/null
+++ b/doc/bugs/Nearline_bucket_stopped_working___40__can__39__t_even_HEAD_files__41__/comment_2_c227071f23a96ed9928f128e7f77e503._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="justin.lebar@7a36fcafc322d9a381e89f08ab6289033c6dde91"
+ nickname="justin.lebar"
+ avatar="http://cdn.libravatar.org/avatar/9fca4b61a1ab555f231851e7543f9a3e"
+ subject="comment 2"
+ date="2016-11-20T03:47:23Z"
+ content="""
+Thanks for your reply, Joey.  Sorry for the delay getting back to this -- I didn't realize I hadn't enabled notifications on the thread.
+
+The GCS docs suggest that 400 errors should be accompanied by an explanation in the reply body.
+
+> Error responses usually include a JSON document in the response body, which contains information about the error.
+
+https://cloud.google.com/storage/docs/json_api/v1/status-codes
+
+Do you think we're not getting an http response body here, or that it's not being printed out?
+"""]]

From d42f5889ef3f22eeab1aeb9740eae96fbc4f35d2 Mon Sep 17 00:00:00 2001
From: "https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
 <https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4@web>
Date: Sun, 20 Nov 2016 04:26:59 +0000
Subject: [PATCH 069/367] initial whining

---
 ..._annex_into_git_in___39__addurl__39__.mdwn | 40 +++++++++++++++++++
 1 file changed, 40 insertions(+)
 create mode 100644 doc/bugs/adds_file_destined_for_annex_into_git_in___39__addurl__39__.mdwn

diff --git a/doc/bugs/adds_file_destined_for_annex_into_git_in___39__addurl__39__.mdwn b/doc/bugs/adds_file_destined_for_annex_into_git_in___39__addurl__39__.mdwn
new file mode 100644
index 0000000000..5b9c76e693
--- /dev/null
+++ b/doc/bugs/adds_file_destined_for_annex_into_git_in___39__addurl__39__.mdwn
@@ -0,0 +1,40 @@
+### Please describe the problem.
+
+When addurl'ing a big file with .gitattributes configured to add only some files directly into git (and 'git annex add' operating correctly), addurl adds large files straight into git.
+
+### What version of git-annex are you using? On what operating system?
+
+git-annex version: 6.20161018+gitgf3c366a-1~ndall+1
+
+
+### Please provide any additional information below.
+
+[[!format sh """
+$> cat .gitattributes 
+* annex.backend=MD5E
+* annex.largefiles=(largerthan=100kb)
+*.json annex.largefiles=nothing
+*.txt annex.largefiles=nothing
+*.tsv annex.largefiles=nothing
+*.nii.gz annex.largefiles=(largerthan=0kb)
+*.tgz annex.largefiles=(largerthan=0kb)
+*.tar.gz annex.largefiles=(largerthan=0kb)
+*.gz annex.largefiles=(largerthan=0kb)
+
+$> git annex addurl http://fcp-indi.s3.amazonaws.com/data/Projects/HBNSSI/RawDataTars/sub-0031121_baseline.tar.gz\?versionId\=7FvexHgyazWF.dUo238FA7XRiK0FWQDw.
+addurl fcp_indi.s3.amazonaws.com_data_Projects_HBNSSI_RawDataTars_sub_0031121_baseline.tar.gz_versionId_7FvexHgyazWF.dUo238FA7XRiK0FWQDw. (downloading http://fcp-indi.s3.amazonaws.com/data/Projects/HBNSSI/RawDataTars/sub-0031121_baseline.tar.gz?versionId=7FvexHgyazWF.dUo238FA7XRiK0FWQDw. ...) 
+/mnt/btrfs/datasets/datalad/crawl-misc/indi/ 100%[==============================================================================================>] 195.44M  21.2MB/s    in 12s     
+(non-large file; adding content to git repository) ok
+(recording state in git...)
+cached/staged changes:                                                                                                                                                              
+ \u2026r.gz_versionId_7FvexHgyazWF.dUo238FA7XRiK0FWQDw. | Bin 0 -> 204937338 bytes
+
+$> ls -l fcp_indi.s3.amazonaws.com_data_Projects_HBNSSI_RawDataTars_sub_0031121_baseline.tar.gz_versionId_7FvexHgyazWF.dUo238FA7XRiK0FWQDw. 
+-rw------- 1 yoh datalad 204937338 Oct 25 17:30 fcp_indi.s3.amazonaws.com_data_Projects_HBNSSI_RawDataTars_sub_0031121_baseline.tar.gz_versionId_7FvexHgyazWF.dUo238FA7XRiK0FWQDw.
+cached/staged changes:                                                                                                                                                              
+ \u2026r.gz_versionId_7FvexHgyazWF.dUo238FA7XRiK0FWQDw. | Bin 0 -> 204937338 bytes
+
+"""]]
+
+
+

From d50b0f3bb3f8509d462b70fbea072227b80e9227 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sun, 20 Nov 2016 12:08:16 -0400
Subject: [PATCH 070/367] implement p2p protocol for Handle

This is most of the way to having the p2p protocol working over tor
hidden services, at least enough to do git push/pull.

The free monad was split into two, one for network operations and the
other for local (Annex) operations. This will allow git-remote-tor-annex
to run only an IO action, not needing the Annex monad.

This commit was sponsored by Remy van Elst on Patreon.
---
 Remote/Helper/P2P.hs    | 572 +++++++++++++++++++++-------------------
 Remote/Helper/P2P/IO.hs | 159 +++++++++++
 git-annex.cabal         |   1 +
 3 files changed, 455 insertions(+), 277 deletions(-)
 create mode 100644 Remote/Helper/P2P/IO.hs

diff --git a/Remote/Helper/P2P.hs b/Remote/Helper/P2P.hs
index d3d3dfa088..fbd6c24635 100644
--- a/Remote/Helper/P2P.hs
+++ b/Remote/Helper/P2P.hs
@@ -7,20 +7,7 @@
 
 {-# LANGUAGE DeriveFunctor, TemplateHaskell, FlexibleContexts, RankNTypes #-}
 
-module Remote.Helper.P2P (
-	AuthToken(..),
-	ProtoF(..),
-	runPure,
-	protoDump,
-	auth,
-	checkPresent,
-	lockContentWhile,
-	remove,
-	get,
-	put,
-	connect,
-	serve,
-) where
+module Remote.Helper.P2P where
 
 import qualified Utility.SimpleProtocol as Proto
 import Types.Key
@@ -33,7 +20,7 @@ import Control.Monad.Free
 import Control.Monad.Free.TH
 import Control.Monad.Catch
 import System.Exit (ExitCode(..))
-import System.IO (Handle)
+import System.IO
 import qualified Data.ByteString.Lazy as L
 
 newtype AuthToken = AuthToken String
@@ -49,10 +36,6 @@ newtype Len = Len Integer
 data Service = UploadPack | ReceivePack
 	deriving (Show)
 
-data RelayData
-	= RelayData L.ByteString
-	| RelayMessage Message
-
 -- | Messages in the protocol. The peer that makes the connection
 -- always initiates requests, and the other peer makes responses to them.
 data Message
@@ -75,264 +58,6 @@ data Message
 	| ERROR String
 	deriving (Show)
 
--- | Free monad for implementing actions that use the protocol.
-data ProtoF next
-	= SendMessage Message next
-	| ReceiveMessage (Message -> next)
-	| SendBytes Len L.ByteString next
-	| ReceiveBytes Len (L.ByteString -> next)
-	-- ^ Lazily reads bytes from peer. Stops once Len are read,
-	-- or if connection is lost, and in either case returns the bytes
-	-- that were read. This allows resuming interrupted transfers.
-	| KeyFileSize Key (Len -> next)
-	-- ^ Checks size of key file (dne = 0)
-	| ReadKeyFile Key Offset (L.ByteString -> next)
-	| WriteKeyFile Key Offset Len L.ByteString (Bool -> next)
-	-- ^ Writes to key file starting at an offset. Returns True
-	-- once the whole content of the key is stored in the key file.
-	--
-	-- Note: The ByteString may not contain the entire remaining content
-	-- of the key. Only once the key file size == Len has the whole
-	-- content been transferred.
-	| CheckAuthToken UUID AuthToken (Bool -> next)
-	| SetPresent Key UUID next
-	| CheckContentPresent Key (Bool -> next)
-	-- ^ Checks if the whole content of the key is locally present.
-	| RemoveKeyFile Key (Bool -> next)
-	-- ^ If the key file is not present, still succeeds.
-	-- May fail if not enough copies to safely drop, etc.
-	| TryLockContent Key (Bool -> Proto ()) next
-	| WriteHandle Handle L.ByteString next
-	-- ^ Try to lock the content of a key,  preventing it
-	-- from being deleted, and run the provided protocol action.
-	| Relay Handle (RelayData -> Proto (Maybe ExitCode)) (ExitCode -> next)
-	-- ^ Waits for data to be written to the Handle, and for messages
-	-- to be received from the peer, and passes the data to the
-	-- callback, continuing until it returns an ExitCode.
-	| RelayService Service
-		(Handle -> RelayData -> Proto (Maybe ExitCode))
-		(ExitCode -> next)
-	-- ^ Runs a service, and waits for it to output to stdout,
-	-- and for messages to be received from the peer, and passes
-	-- the data to the callback (which is also passed the service's
-	-- stdin Handle), continuing uniil the service exits.
-	deriving (Functor)
-
-type Proto = Free ProtoF
-
-$(makeFree ''ProtoF)
-
--- | Running Proto actions purely, to see what they do.
-runPure :: Show r => Proto r -> [Message] -> [(String, Maybe Message)]
-runPure (Pure r) _ = [("result: " ++ show r, Nothing)]
-runPure (Free (SendMessage m next)) ms = (">",  Just m):runPure next ms
-runPure (Free (ReceiveMessage _)) [] = [("not enough Messages provided", Nothing)]
-runPure (Free (ReceiveMessage next)) (m:ms) = ("<", Just m):runPure (next m) ms
-runPure (Free (SendBytes _ _ next)) ms = ("> bytes", Nothing):runPure next ms
-runPure (Free (ReceiveBytes _ next)) ms = ("< bytes", Nothing):runPure (next L.empty) ms
-runPure (Free (KeyFileSize _ next)) ms = runPure (next (Len 100)) ms
-runPure (Free (ReadKeyFile _ _ next)) ms = runPure (next L.empty) ms
-runPure (Free (WriteKeyFile _ _ _ _ next)) ms = runPure (next True) ms
-runPure (Free (CheckAuthToken _ _ next)) ms = runPure (next True) ms
-runPure (Free (SetPresent _ _ next)) ms = runPure next ms
-runPure (Free (CheckContentPresent _ next)) ms = runPure (next False) ms
-runPure (Free (RemoveKeyFile _ next)) ms = runPure (next True) ms
-runPure (Free (TryLockContent _ p next)) ms = runPure (p True >> next) ms
-runPure (Free (WriteHandle _ _ next)) ms = runPure next ms
-runPure (Free (Relay _ _ next)) ms = runPure (next ExitSuccess) ms
-runPure (Free (RelayService _ _ next)) ms = runPure (next ExitSuccess) ms
-
-protoDump :: [(String, Maybe Message)] -> String
-protoDump = unlines . map protoDump'
-
-protoDump' :: (String, Maybe Message) -> String
-protoDump' (s, Nothing) = s
-protoDump' (s, Just m) = s ++ " " ++ unwords (Proto.formatMessage m)
-
-auth :: UUID -> AuthToken -> Proto (Maybe UUID)
-auth myuuid t = do
-	sendMessage (AUTH myuuid t)
-	r <- receiveMessage
-	case r of
-		AUTH_SUCCESS theiruuid -> return $ Just theiruuid
-		AUTH_FAILURE -> return Nothing
-		_ -> do
-			sendMessage (ERROR "auth failed")
-			return Nothing
-
-checkPresent :: Key -> Proto Bool
-checkPresent key = do
-	sendMessage (CHECKPRESENT key)
-	checkSuccess
-
-{- Locks content to prevent it from being dropped, while running an action.
- -
- - Note that this only guarantees that the content is locked as long as the
- - connection to the peer remains up. If the connection is unexpectededly
- - dropped, the peer will then unlock the content.
- -}
-lockContentWhile 
-	:: MonadMask m 
-	=> (forall r. Proto r -> m r)
-	-> Key
-	-> (Bool -> m ())
-	-> m ()
-lockContentWhile runproto key a = bracket setup cleanup a
-  where
-	setup = runproto $ do
-		sendMessage (LOCKCONTENT key)
-		checkSuccess
-	cleanup True = runproto $ sendMessage UNLOCKCONTENT
-	cleanup False = return ()
-
-remove :: Key -> Proto Bool
-remove key = do
-	sendMessage (REMOVE key)
-	checkSuccess
-
-get :: Key -> Proto Bool
-get key = receiveContent key (`GET` key)
-
-put :: Key -> Proto Bool
-put key = do
-	sendMessage (PUT key)
-	r <- receiveMessage
-	case r of
-		PUT_FROM offset -> sendContent key offset
-		ALREADY_HAVE -> return True
-		_ -> do
-			sendMessage (ERROR "expected PUT_FROM")
-			return False
-
-connect :: Service -> Handle -> Handle -> Proto ExitCode
-connect service hin hout = do
-	sendMessage (CONNECT service)
-	relay hin (relayCallback hout)
-
-relayCallback :: Handle -> RelayData -> Proto (Maybe ExitCode)
-relayCallback hout (RelayMessage (DATA len)) = do
-	writeHandle hout =<< receiveBytes len
-	return Nothing
-relayCallback _ (RelayMessage (CONNECTDONE exitcode)) =
-	return (Just exitcode)
-relayCallback _ (RelayMessage _) = do
-	sendMessage (ERROR "expected DATA or CONNECTDONE")
-	return (Just (ExitFailure 1))
-relayCallback _ (RelayData b) = do
-	let len = Len $ fromIntegral $ L.length b
-	sendMessage (DATA len)
-	sendBytes len b
-	return Nothing
-
--- | Serve the protocol.
---
--- Note that if the client sends an unexpected message, the server will
--- respond with PTOTO_ERROR, and always continues processing messages.
--- Since the protocol is not versioned, this is necessary to handle
--- protocol changes robustly, since the client can detect when it's
--- talking to a server that does not support some new feature, and fall
--- back.
---
--- When the client sends ERROR to the server, the server gives up,
--- since it's not clear what state the client is is, and so not possible to
--- recover.
-serve :: UUID -> Proto ()
-serve myuuid = go Nothing
-  where
-	go autheduuid = do
-		r <- receiveMessage
-		case r of
-			AUTH theiruuid authtoken -> do
-				ok <- checkAuthToken theiruuid authtoken
-				if ok
-					then do
-						sendMessage (AUTH_SUCCESS myuuid)
-						go (Just theiruuid)
-					else do
-						sendMessage AUTH_FAILURE
-						go autheduuid
-			ERROR _ -> return ()
-			_ -> do
-				case autheduuid of
-					Just theiruuid -> authed theiruuid r
-					Nothing -> sendMessage (ERROR "must AUTH first")
-				go autheduuid
-	
-	authed _theiruuid r = case r of
-		LOCKCONTENT key -> tryLockContent key $ \locked -> do
-			sendSuccess locked
-			when locked $ do
-				r' <- receiveMessage
-				case r' of
-					UNLOCKCONTENT -> return ()
-					_ -> sendMessage (ERROR "expected UNLOCKCONTENT")
-		CHECKPRESENT key -> sendSuccess =<< checkContentPresent key
-		REMOVE key -> sendSuccess =<< removeKeyFile key
-		PUT key -> do
-			have <- checkContentPresent key
-			if have
-				then sendMessage ALREADY_HAVE
-				else do
-					ok <- receiveContent key PUT_FROM
-					when ok $
-						setPresent key myuuid
-		-- setPresent not called because the peer may have
-		-- requested the data but not permanatly stored it.
-		GET offset key -> void $ sendContent key offset
-		CONNECT service -> do
-			exitcode <- relayService service relayCallback
-			sendMessage (CONNECTDONE exitcode)
-		_ -> sendMessage (ERROR "unexpected command")
-
-sendContent :: Key -> Offset -> Proto Bool
-sendContent key offset = do
-	(len, content) <- readKeyFileLen key offset
-	sendMessage (DATA len)
-	sendBytes len content
-	checkSuccess
-
-receiveContent :: Key -> (Offset -> Message) -> Proto Bool
-receiveContent key mkmsg = do
-	Len n <- keyFileSize key
-	let offset = Offset n
-	sendMessage (mkmsg offset)
-	r <- receiveMessage
-	case r of
-		DATA len -> do
-			ok <- writeKeyFile key offset len =<< receiveBytes len
-			sendSuccess ok
-			return ok
-		_ -> do
-			sendMessage (ERROR "expected DATA")
-			return False
-
-checkSuccess :: Proto Bool
-checkSuccess = do
-	ack <- receiveMessage
-	case ack of
-		SUCCESS -> return True
-		FAILURE -> return False
-		_ -> do
-			sendMessage (ERROR "expected SUCCESS or FAILURE")
-			return False
-
-sendSuccess :: Bool -> Proto ()
-sendSuccess True = sendMessage SUCCESS
-sendSuccess False = sendMessage FAILURE
-
--- Reads key file from an offset. The Len should correspond to
--- the length of the ByteString, but to avoid buffering the content
--- in memory, is gotten using keyFileSize.
-readKeyFileLen :: Key -> Offset -> Proto (Len, L.ByteString)
-readKeyFileLen key (Offset offset) = do
-	(Len totallen) <- keyFileSize key
-	let len = totallen - offset
-	if len <= 0
-		then return (Len 0, L.empty)
-		else do
-			content <- readKeyFile key (Offset offset)
-			return (Len len, content)
-
 instance Proto.Sendable Message where
 	formatMessage (AUTH uuid authtoken) = ["AUTH", Proto.serialize uuid, Proto.serialize authtoken]
 	formatMessage (AUTH_SUCCESS uuid) = ["AUTH-SUCCESS",  Proto.serialize uuid]
@@ -390,3 +115,296 @@ instance Proto.Serializable Service where
 	deserialize "git-upload-pack" = Just UploadPack
 	deserialize "git-receive-pack" = Just ReceivePack
 	deserialize _ = Nothing
+
+-- | Free monad for the protocol, combining net communication,
+-- and local actions.
+data ProtoF c = Net (NetF c) | Local (LocalF c)
+	deriving (Functor)
+
+type Proto = Free ProtoF
+
+net :: Net a -> Proto a
+net = hoistFree Net
+
+local :: Local a -> Proto a
+local = hoistFree Local
+
+data NetF c
+	= SendMessage Message c
+	| ReceiveMessage (Message -> c)
+	| SendBytes Len L.ByteString c
+	| ReceiveBytes Len (L.ByteString -> c)
+	| Relay RelayHandle
+		(RelayData -> Net (Maybe ExitCode))
+		(ExitCode -> c)
+	-- ^ Waits for data to be written to the RelayHandle, and for messages
+	-- to be received from the peer, and passes the data to the
+	-- callback, continuing until it returns an ExitCode.
+	| RelayService Service
+		(RelayHandle -> RelayData -> Net (Maybe ExitCode))
+		(ExitCode -> c)
+	-- ^ Runs a service, and waits for it to output to stdout,
+	-- and for messages to be received from the peer, and passes
+	-- the data to the callback (which is also passed the service's
+	-- stdin RelayHandle), continuing uniil the service exits.
+	| WriteRelay RelayHandle L.ByteString c
+	-- ^ Write data to a relay's handle, flushing it immediately.
+	deriving (Functor)
+
+type Net = Free NetF
+
+data RelayData
+	= RelayData L.ByteString
+	| RelayMessage Message
+
+newtype RelayHandle = RelayHandle Handle
+
+data LocalF c
+	-- ^ Lazily reads bytes from peer. Stops once Len are read,
+	-- or if connection is lost, and in either case returns the bytes
+	-- that were read. This allows resuming interrupted transfers.
+	= KeyFileSize Key (Len -> c)
+	-- ^ Checks size of key file (dne = 0)
+	| ReadKeyFile Key Offset (L.ByteString -> c)
+	| WriteKeyFile Key Offset Len L.ByteString (Bool -> c)
+	-- ^ Writes to key file starting at an offset. Returns True
+	-- once the whole content of the key is stored in the key file.
+	--
+	-- Note: The ByteString may not contain the entire remaining content
+	-- of the key. Only once the key file size == Len has the whole
+	-- content been transferred.
+	| CheckAuthToken UUID AuthToken (Bool -> c)
+	| SetPresent Key UUID c
+	| CheckContentPresent Key (Bool -> c)
+	-- ^ Checks if the whole content of the key is locally present.
+	| RemoveKeyFile Key (Bool -> c)
+	-- ^ If the key file is not present, still succeeds.
+	-- May fail if not enough copies to safely drop, etc.
+	| TryLockContent Key (Bool -> Proto ()) c
+	-- ^ Try to lock the content of a key,  preventing it
+	-- from being deleted, and run the provided protocol action.
+	deriving (Functor)
+
+type Local = Free LocalF
+
+-- Generate sendMessage etc functions for all free monad constructors.
+$(makeFree ''NetF)
+$(makeFree ''LocalF)
+
+-- | Running Proto actions purely, to see what they do.
+runPure :: Show r => Proto r -> [Message] -> [(String, Maybe Message)]
+runPure (Pure r) _ = [("result: " ++ show r, Nothing)]
+runPure (Free (Net n)) ms = runNet n ms
+runPure (Free (Local n)) ms = runLocal n ms
+
+runNet :: Show r => NetF (Proto r) -> [Message] -> [(String, Maybe Message)]
+runNet (SendMessage m next) ms = (">",  Just m):runPure next ms
+runNet (ReceiveMessage _) [] = [("not enough Messages provided", Nothing)]
+runNet (ReceiveMessage next) (m:ms) = ("<", Just m):runPure (next m) ms
+runNet (SendBytes _ _ next) ms = ("> bytes", Nothing):runPure next ms
+runNet (ReceiveBytes _ next) ms = ("< bytes", Nothing):runPure (next L.empty) ms
+runNet (Relay _ _ next) ms = runPure (next ExitSuccess) ms
+runNet (RelayService _ _ next) ms = runPure (next ExitSuccess) ms
+runNet (WriteRelay _ _ next) ms = runPure next ms
+
+runLocal :: Show r => LocalF (Proto r) -> [Message] -> [(String, Maybe Message)]
+runLocal (KeyFileSize _ next) ms = runPure (next (Len 100)) ms
+runLocal (ReadKeyFile _ _ next) ms = runPure (next L.empty) ms
+runLocal (WriteKeyFile _ _ _ _ next) ms = runPure (next True) ms
+runLocal (CheckAuthToken _ _ next) ms = runPure (next True) ms
+runLocal (SetPresent _ _ next) ms = runPure next ms
+runLocal (CheckContentPresent _ next) ms = runPure (next False) ms
+runLocal (RemoveKeyFile _ next) ms = runPure (next True) ms
+runLocal (TryLockContent _ p next) ms = runPure (p True >> next) ms
+
+protoDump :: [(String, Maybe Message)] -> String
+protoDump = unlines . map protoDump'
+
+protoDump' :: (String, Maybe Message) -> String
+protoDump' (s, Nothing) = s
+protoDump' (s, Just m) = s ++ " " ++ unwords (Proto.formatMessage m)
+
+auth :: UUID -> AuthToken -> Proto (Maybe UUID)
+auth myuuid t = do
+	net $ sendMessage (AUTH myuuid t)
+	r <- net receiveMessage
+	case r of
+		AUTH_SUCCESS theiruuid -> return $ Just theiruuid
+		AUTH_FAILURE -> return Nothing
+		_ -> do
+			net $ sendMessage (ERROR "auth failed")
+			return Nothing
+
+checkPresent :: Key -> Proto Bool
+checkPresent key = do
+	net $ sendMessage (CHECKPRESENT key)
+	checkSuccess
+
+{- Locks content to prevent it from being dropped, while running an action.
+ -
+ - Note that this only guarantees that the content is locked as long as the
+ - connection to the peer remains up. If the connection is unexpectededly
+ - dropped, the peer will then unlock the content.
+ -}
+lockContentWhile 
+	:: MonadMask m 
+	=> (forall r. Proto r -> m r)
+	-> Key
+	-> (Bool -> m ())
+	-> m ()
+lockContentWhile runproto key a = bracket setup cleanup a
+  where
+	setup = runproto $ do
+		net $ sendMessage (LOCKCONTENT key)
+		checkSuccess
+	cleanup True = runproto $ net $ sendMessage UNLOCKCONTENT
+	cleanup False = return ()
+
+remove :: Key -> Proto Bool
+remove key = do
+	net $ sendMessage (REMOVE key)
+	checkSuccess
+
+get :: Key -> Proto Bool
+get key = receiveContent key (`GET` key)
+
+put :: Key -> Proto Bool
+put key = do
+	net $ sendMessage (PUT key)
+	r <- net receiveMessage
+	case r of
+		PUT_FROM offset -> sendContent key offset
+		ALREADY_HAVE -> return True
+		_ -> do
+			net $ sendMessage (ERROR "expected PUT_FROM")
+			return False
+
+-- | Serve the protocol.
+--
+-- Note that if the client sends an unexpected message, the server will
+-- respond with PTOTO_ERROR, and always continues processing messages.
+-- Since the protocol is not versioned, this is necessary to handle
+-- protocol changes robustly, since the client can detect when it's
+-- talking to a server that does not support some new feature, and fall
+-- back.
+--
+-- When the client sends ERROR to the server, the server gives up,
+-- since it's not clear what state the client is is, and so not possible to
+-- recover.
+serve :: UUID -> Proto ()
+serve myuuid = go Nothing
+  where
+	go autheduuid = do
+		r <- net receiveMessage
+		case r of
+			AUTH theiruuid authtoken -> do
+				ok <- local $ checkAuthToken theiruuid authtoken
+				if ok
+					then do
+						net $ sendMessage (AUTH_SUCCESS myuuid)
+						go (Just theiruuid)
+					else do
+						net $ sendMessage AUTH_FAILURE
+						go autheduuid
+			ERROR _ -> return ()
+			_ -> do
+				case autheduuid of
+					Just theiruuid -> authed theiruuid r
+					Nothing -> net $ sendMessage (ERROR "must AUTH first")
+				go autheduuid
+	
+	authed _theiruuid r = case r of
+		LOCKCONTENT key -> local $ tryLockContent key $ \locked -> do
+			sendSuccess locked
+			when locked $ do
+				r' <- net receiveMessage
+				case r' of
+					UNLOCKCONTENT -> return ()
+					_ -> net $ sendMessage (ERROR "expected UNLOCKCONTENT")
+		CHECKPRESENT key -> sendSuccess =<< local (checkContentPresent key)
+		REMOVE key ->  sendSuccess =<< local (removeKeyFile key)
+		PUT key -> do
+			have <- local $ checkContentPresent key
+			if have
+				then net $ sendMessage ALREADY_HAVE
+				else do
+					ok <- receiveContent key PUT_FROM
+					when ok $
+						local $ setPresent key myuuid
+		-- setPresent not called because the peer may have
+		-- requested the data but not permanatly stored it.
+		GET offset key -> void $ sendContent key offset
+		CONNECT service -> do
+			exitcode <- net $ relayService service relayCallback
+			net $ sendMessage (CONNECTDONE exitcode)
+		_ -> net $ sendMessage (ERROR "unexpected command")
+
+sendContent :: Key -> Offset -> Proto Bool
+sendContent key offset = do
+	(len, content) <- readKeyFileLen key offset
+	net $ sendMessage (DATA len)
+	net $ sendBytes len content
+	checkSuccess
+
+receiveContent :: Key -> (Offset -> Message) -> Proto Bool
+receiveContent key mkmsg = do
+	Len n <- local $ keyFileSize key
+	let offset = Offset n
+	net $ sendMessage (mkmsg offset)
+	r <- net receiveMessage
+	case r of
+		DATA len -> do
+			ok <- local . writeKeyFile key offset len
+				=<< net (receiveBytes len)
+			sendSuccess ok
+			return ok
+		_ -> do
+			net $ sendMessage (ERROR "expected DATA")
+			return False
+
+checkSuccess :: Proto Bool
+checkSuccess = do
+	ack <- net receiveMessage
+	case ack of
+		SUCCESS -> return True
+		FAILURE -> return False
+		_ -> do
+			net $ sendMessage (ERROR "expected SUCCESS or FAILURE")
+			return False
+
+sendSuccess :: Bool -> Proto ()
+sendSuccess True = net $ sendMessage SUCCESS
+sendSuccess False = net $ sendMessage FAILURE
+
+-- Reads key file from an offset. The Len should correspond to
+-- the length of the ByteString, but to avoid buffering the content
+-- in memory, is gotten using keyFileSize.
+readKeyFileLen :: Key -> Offset -> Proto (Len, L.ByteString)
+readKeyFileLen key (Offset offset) = do
+	(Len totallen) <- local $ keyFileSize key
+	let len = totallen - offset
+	if len <= 0
+		then return (Len 0, L.empty)
+		else do
+			content <- local $ readKeyFile key (Offset offset)
+			return (Len len, content)
+
+connect :: Service -> Handle -> Handle -> Proto ExitCode
+connect service hin hout = do
+	net $ sendMessage (CONNECT service)
+	net $ relay (RelayHandle hin) (relayCallback (RelayHandle hout))
+
+relayCallback :: RelayHandle -> RelayData -> Net (Maybe ExitCode)
+relayCallback hout (RelayMessage (DATA len)) = do
+	writeRelay hout =<< receiveBytes len
+	return Nothing
+relayCallback _ (RelayMessage (CONNECTDONE exitcode)) =
+	return (Just exitcode)
+relayCallback _ (RelayMessage _) = do
+	sendMessage (ERROR "expected DATA or CONNECTDONE")
+	return (Just (ExitFailure 1))
+relayCallback _ (RelayData b) = do
+	let len = Len $ fromIntegral $ L.length b
+	sendMessage (DATA len)
+	sendBytes len b
+	return Nothing
diff --git a/Remote/Helper/P2P/IO.hs b/Remote/Helper/P2P/IO.hs
new file mode 100644
index 0000000000..7179adc2b5
--- /dev/null
+++ b/Remote/Helper/P2P/IO.hs
@@ -0,0 +1,159 @@
+{- P2P protocol, partial IO implementation
+ -
+ - Copyright 2016 Joey Hess <id@joeyh.name>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE RankNTypes #-}
+
+module Remote.Helper.P2P.IO
+	( RunProto
+	, runProtoHandle
+	) where
+
+import Remote.Helper.P2P
+import Utility.Process
+import Git
+import Git.Command
+import Utility.SafeCommand
+import Utility.SimpleProtocol
+
+import Control.Monad
+import Control.Monad.Free
+import Control.Monad.IO.Class
+import Data.Maybe
+import System.Exit (ExitCode(..))
+import System.IO
+import Control.Concurrent
+import qualified Data.ByteString as B
+import qualified Data.ByteString.Lazy as L
+
+type RunProto = forall a m. MonadIO m => Proto a -> m a
+
+data S = S
+	{ repo :: Repo
+	, hdl :: Handle
+	}
+
+-- Implementation of the protocol, communicating with a peer
+-- over a Handle. No Local actions will be run.
+runProtoHandle :: MonadIO m => Handle -> Repo -> Proto a -> m a
+runProtoHandle h r = go
+  where
+	go :: RunProto
+	go (Pure a) = pure a
+	go (Free (Net n)) = runNetHandle (S r h) go n
+	go (Free (Local _)) = error "local actions not allowed"
+
+runNetHandle :: MonadIO m => S -> RunProto -> NetF (Proto a) -> m a
+runNetHandle s runner f = case f of
+	SendMessage m next -> do
+		liftIO $ do
+			hPutStrLn (hdl s) (unwords (formatMessage m))
+			hFlush (hdl s)
+		runner next
+	ReceiveMessage next -> do
+		l <- liftIO $ hGetLine (hdl s)
+		let m = fromMaybe (ERROR "protocol parse error")
+			(parseMessage l)
+		runner (next m)
+	SendBytes _len b next -> do
+		liftIO $ do
+			L.hPut (hdl s) b
+			hFlush (hdl s)
+		runner next
+	ReceiveBytes (Len n) next -> do
+		b <- liftIO $ L.hGet (hdl s) (fromIntegral n)
+		runner (next b)
+	Relay hout callback next ->
+		runRelay runner hout callback >>= runner . next
+	RelayService service callback next ->
+		runRelayService s runner service callback >>= runner . next
+	WriteRelay (RelayHandle h) b next -> do
+		liftIO $ do
+			L.hPut h b
+			hFlush h
+		runner next
+
+runRelay
+	:: MonadIO m
+	=> RunProto
+	-> RelayHandle
+	-> (RelayData -> Net (Maybe ExitCode))
+	-> m ExitCode
+runRelay runner (RelayHandle hout) callback = do
+	v <- liftIO newEmptyMVar
+	_ <- liftIO $ forkIO $ readout v
+	feeder <- liftIO $ forkIO $ feedin v
+	exitcode <- liftIO $ drain v
+	liftIO $ killThread feeder
+	return exitcode
+  where
+	feedin v = forever $ do
+		m <- runner $ net receiveMessage
+		putMVar v $ RelayMessage m
+	
+	readout v = do
+		b <- B.hGetSome hout 65536
+		if B.null b
+			then hClose hout
+			else do
+				putMVar v $ RelayData (L.fromChunks [b])
+				readout v
+
+	drain v = do
+		d <- takeMVar v
+		r <- runner $ net $ callback d
+		case r of
+			Nothing -> drain v
+			Just exitcode -> return exitcode
+
+runRelayService
+	:: MonadIO m
+	=> S
+	-> RunProto
+	-> Service
+	-> (RelayHandle -> RelayData -> Net (Maybe ExitCode))
+	-> m ExitCode
+runRelayService s runner service callback = do
+	v <- liftIO newEmptyMVar
+	(Just hin, Just hout, _, pid) <- liftIO $ createProcess serviceproc
+		{ std_out = CreatePipe
+		, std_in = CreatePipe
+		}
+	_ <- liftIO $ forkIO $ readout v hout
+	feeder <- liftIO $ forkIO $ feedin v
+	_ <- liftIO $ forkIO $ putMVar v . Left =<< waitForProcess pid
+	exitcode <- liftIO $ drain v hin
+	liftIO $ killThread feeder
+	return exitcode
+  where
+	cmd = case service of
+		UploadPack -> "upload-pack"
+		ReceivePack -> "receive-pack"
+	serviceproc = gitCreateProcess [Param cmd, File (repoPath (repo s))] (repo s)
+
+	drain v hin = do
+		d <- takeMVar v
+		case d of
+			Left exitcode -> do
+				hClose hin
+				return exitcode
+			Right relaydata -> do
+				_ <- runner $ net $
+					callback (RelayHandle hin) relaydata
+				drain v hin
+	
+	readout v hout = do
+		b <- B.hGetSome hout 65536
+		if B.null b
+			then hClose hout
+			else do
+				putMVar v $ Right $ 
+					RelayData (L.fromChunks [b])
+				readout v hout
+
+	feedin v = forever $ do
+		m <- runner $ net receiveMessage
+		putMVar v $ Right $ RelayMessage m
diff --git a/git-annex.cabal b/git-annex.cabal
index 4fb4e1c3c0..77c50b66e4 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -920,6 +920,7 @@ Executable git-annex
     Remote.Helper.Http
     Remote.Helper.Messages
     Remote.Helper.P2P
+    Remote.Helper.P2P.IO
     Remote.Helper.ReadOnly
     Remote.Helper.Special
     Remote.Helper.Ssh

From a101b8de37e7be84e2b2c19417926a6f08f1e2a2 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sun, 20 Nov 2016 14:39:26 -0400
Subject: [PATCH 071/367] remotedaemon: Fork to background by default. Added
 --foreground switch to enable old behavior.

Groundwork for tor hidden services, which the remotedaemon will serve.
---
 Assistant/Threads/RemoteControl.hs |  2 +-
 CHANGELOG                          |  7 +++++
 Command/EnableTor.hs               |  2 +-
 Command/RemoteDaemon.hs            | 31 +++++++++++++---------
 RemoteDaemon/Core.hs               | 26 ++++++++++++++-----
 doc/git-annex-enable-tor.mdwn      |  5 ++++
 doc/git-annex-remotedaemon.mdwn    | 41 ++++++++++++++++++++++--------
 doc/git-annex.mdwn                 | 19 +++++++++-----
 8 files changed, 97 insertions(+), 36 deletions(-)

diff --git a/Assistant/Threads/RemoteControl.hs b/Assistant/Threads/RemoteControl.hs
index 447b493c65..1aa8bc9c8b 100644
--- a/Assistant/Threads/RemoteControl.hs
+++ b/Assistant/Threads/RemoteControl.hs
@@ -30,7 +30,7 @@ remoteControlThread :: NamedThread
 remoteControlThread = namedThread "RemoteControl" $ do
 	program <- liftIO programPath
 	(cmd, params) <- liftIO $ toBatchCommand
-		(program, [Param "remotedaemon"])
+		(program, [Param "remotedaemon", Param "--foreground"])
 	let p = proc cmd (toCommand params)
 	(Just toh, Just fromh, _, pid) <- liftIO $ createProcess p
 		{ std_in = CreatePipe
diff --git a/CHANGELOG b/CHANGELOG
index 3777e6d5ad..692a22ea47 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,10 @@
+git-annex (6.20161119) UNRELEASED; urgency=medium
+
+  * remotedaemon: Fork to background by default. Added --foreground switch
+    to enable old behavior.
+
+ -- Joey Hess <id@joeyh.name>  Sun, 20 Nov 2016 14:10:15 -0400
+
 git-annex (6.20161118) unstable; urgency=medium
 
   * git-annex.cabal: Loosen bounds on persistent to allow 2.5, which
diff --git a/Command/EnableTor.hs b/Command/EnableTor.hs
index 1a54c6c5d8..369ea75093 100644
--- a/Command/EnableTor.hs
+++ b/Command/EnableTor.hs
@@ -14,7 +14,7 @@ import Utility.Tor
 -- git-annex, as that would create root-owned files.
 cmd :: Command
 cmd = noCommit $ dontCheck repoExists $
-	command "enable-tor" SectionPlumbing ""
+	command "enable-tor" SectionSetup ""
 		"userid uuid" (withParams seek)
 
 seek :: CmdParams -> CommandSeek
diff --git a/Command/RemoteDaemon.hs b/Command/RemoteDaemon.hs
index 7c7ecef4bc..c68cf816a3 100644
--- a/Command/RemoteDaemon.hs
+++ b/Command/RemoteDaemon.hs
@@ -1,25 +1,32 @@
 {- git-annex command
  -
- - Copyright 2014 Joey Hess <id@joeyh.name>
+ - Copyright 2014-2016 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
+{-# LANGUAGE CPP #-}
+
 module Command.RemoteDaemon where
 
 import Command
 import RemoteDaemon.Core
+import Utility.Daemon
 
 cmd :: Command
-cmd = noCommit $ 
-	command "remotedaemon" SectionPlumbing
-		"detects when remotes have changed, and fetches from them"
-		paramNothing (withParams seek)
+cmd = noCommit $ dontCheck repoExists $
+	command "remotedaemon" SectionMaintenance
+		"persistent communication with remotes"
+		paramNothing (run <$$> const parseDaemonOptions)
 
-seek :: CmdParams -> CommandSeek
-seek = withNothing start
-
-start :: CommandStart
-start = do
-	liftIO runForeground
-	stop
+run :: DaemonOptions -> CommandSeek
+run o
+	| stopDaemonOption o = error "--stop not implemented for remotedaemon"
+	| foregroundDaemonOption o = liftIO runInteractive
+	| otherwise = do
+#ifndef mingw32_HOST_OS
+		nullfd <- liftIO $ openFd "/dev/null" ReadOnly Nothing defaultFileFlags
+		liftIO $ daemonize nullfd Nothing False runNonInteractive
+#else
+		liftIO $ foreground Nothing runNonInteractive	
+#endif
diff --git a/RemoteDaemon/Core.hs b/RemoteDaemon/Core.hs
index 5fa4131553..3b3f6d98d7 100644
--- a/RemoteDaemon/Core.hs
+++ b/RemoteDaemon/Core.hs
@@ -1,11 +1,11 @@
 {- git-remote-daemon core
  -
- - Copyright 2014 Joey Hess <id@joeyh.name>
+ - Copyright 2014-2016 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module RemoteDaemon.Core (runForeground) where
+module RemoteDaemon.Core (runInteractive, runNonInteractive) where
 
 import qualified Annex
 import Common
@@ -17,6 +17,7 @@ import qualified Git
 import qualified Git.Types as Git
 import qualified Git.CurrentRepo
 import Utility.SimpleProtocol
+import Utility.ThreadScheduler
 import Config
 import Annex.Ssh
 
@@ -26,8 +27,8 @@ import Control.Concurrent.STM
 import Network.URI
 import qualified Data.Map as M
 
-runForeground :: IO ()
-runForeground = do
+runInteractive :: IO ()
+runInteractive = do
 	(readh, writeh) <- dupIoHandles
 	ichan <- newTChanIO :: IO (TChan Consumed)
 	ochan <- newTChanIO :: IO (TChan Emitted)
@@ -44,8 +45,21 @@ runForeground = do
 	let controller = runController ichan ochan
 	
 	-- If any thread fails, the rest will be killed.
-	void $ tryIO $
-		reader `concurrently` writer `concurrently` controller
+	void $ tryIO $ reader `concurrently` writer `concurrently` controller
+
+runNonInteractive :: IO ()
+runNonInteractive = do
+	ichan <- newTChanIO :: IO (TChan Consumed)
+	ochan <- newTChanIO :: IO (TChan Emitted)
+	
+	let reader = forever $ do
+		threadDelaySeconds (Seconds (60*60))
+		atomically $ writeTChan ichan RELOAD
+	let writer = forever $
+		void $ atomically $ readTChan ochan
+	let controller = runController ichan ochan
+	
+	void $ tryIO $ reader `concurrently` writer `concurrently` controller
 
 type RemoteMap = M.Map Git.Repo (IO (), TChan Consumed)
 
diff --git a/doc/git-annex-enable-tor.mdwn b/doc/git-annex-enable-tor.mdwn
index b44cf817cf..5355eef8b3 100644
--- a/doc/git-annex-enable-tor.mdwn
+++ b/doc/git-annex-enable-tor.mdwn
@@ -14,10 +14,15 @@ It outputs to stdout a line of the form "address.onion:onionport socketfile"
 
 This command has to be run by root, since it modifies `/etc/tor/torrc`.
 
+After this command is run, `git annex remotedaemon` can be run to serve the
+hidden service.
+
 # SEE ALSO
 
 [[git-annex]](1)
 
+[[git-annex-remotedaemon]](1)
+
 # AUTHOR
 
 Joey Hess <id@joeyh.name>
diff --git a/doc/git-annex-remotedaemon.mdwn b/doc/git-annex-remotedaemon.mdwn
index 69b516283f..71dd32d309 100644
--- a/doc/git-annex-remotedaemon.mdwn
+++ b/doc/git-annex-remotedaemon.mdwn
@@ -1,6 +1,6 @@
 # NAME
 
-git-annex remotedaemon - detects when remotes have changed, and fetches from them
+git-annex remotedaemon - persistent communication with remotes
 
 # SYNOPSIS
 
@@ -8,18 +8,37 @@ git annex remotedaemon
 
 # DESCRIPTION
 
-This plumbing-level command is used by the assistant to detect
-when remotes have received git pushes, so the changes can be promptly
-fetched and the local repository updated.
+The remotedaemon provides persistent communication with remotes.
+This is useful to detect when remotes have received git pushes, so the
+changes can be promptly fetched and the local repository updated.
 
-This is a better alternative to the [[git-annex-xmppgit]](1)
-hack.
+The assistant runs the remotedaemon and communicates with it on
+stdio using a simple textual protocol.
 
-For the remotedaemon to work, the git remote must have
-[[git-annex-shell]](1) installed, with notifychanges support.
-The first version of git-annex-shell that supports it is 5.20140405.
+Several types of remotes are supported:
 
-It's normal for this process to be running when the assistant is running.
+For ssh remotes, the remotedaemon tries to maintain a connection to the
+remote git repository, and uses git-annex-shell notifychanges to detect
+when the remote git repository has changed, and fetch the changes from
+it. For this to work, the git remote must have [[git-annex-shell]](1)
+installed, with notifychanges support. The first version of git-annex-shell
+that supports it is 5.20140405.
+
+For tor-annex remotes, the remotedaemon runs as a tor hidden service,
+accepting connections from other nodes and serving up the contents of the
+repository. This is only done if you first run `git annex enable-tor`.
+
+# OPTIONS
+
+* `--foreground`
+
+Don't fork to the background, and communicate on stdin/stdout using a
+simple textual protocol. The assistant runs the remotedaemon this way.
+
+Commands in the protocol include LOSTNET, which tells the remotedaemon
+that the network connection has been lost, and causes it to stop any TCP
+connctions. That can be followed by RESUME when the network connection
+comes back up.
 
 # SEE ALSO
 
@@ -27,6 +46,8 @@ It's normal for this process to be running when the assistant is running.
 
 [[git-annex-assistant]](1)
 
+[[git-annex-enable-tor]](1)
+
 # AUTHOR
 
 Joey Hess <id@joeyh.name>
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 955f67629f..773e1b8176 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -212,6 +212,12 @@ subdirectories).
   
   See [[git-annex-enableremote]](1) for details.
 
+* `enable-tor`
+
+  Sets up tor hidden service.
+  
+  See [[git-annex-enable-tor]](1) for details.
+
 * `numcopies [N]`
 
   Configure desired number of copies.
@@ -379,6 +385,13 @@ subdirectories).
   
   See [[git-annex-repair]](1) for details.
 
+* `remotedaemon`
+
+  Persistent communication with remotes.
+
+  See [[git-annex-remotedaemon]](1) for details.
+
+
 # QUERY COMMANDS
 
 * `find [path ...]`
@@ -652,12 +665,6 @@ subdirectories).
 
   See [[git-annex-smudge]](1) for details.
 
-* `remotedaemon`
-
-  Detects when network remotes have received git pushes and fetches from them.
-
-  See [[git-annex-remotedaemon]](1) for details.
-
 * `xmppgit`
 
   This command is used internally by the assistant to perform git pulls

From 74691ddf0edab80df65b09bc3d3224494e9192d6 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sun, 20 Nov 2016 15:45:01 -0400
Subject: [PATCH 072/367] remotedaemon: serve tor hidden service

---
 CHANGELOG                     |  2 ++
 Command/RemoteDaemon.hs       |  2 +-
 Remote/Helper/P2P/IO.hs       |  6 ++---
 RemoteDaemon/Core.hs          |  9 +++++--
 RemoteDaemon/Transport.hs     |  4 +++
 RemoteDaemon/Transport/Tor.hs | 51 +++++++++++++++++++++++++++++++++++
 Utility/Tor.hs                | 19 +++++++++----
 git-annex.cabal               |  1 +
 8 files changed, 83 insertions(+), 11 deletions(-)
 create mode 100644 RemoteDaemon/Transport/Tor.hs

diff --git a/CHANGELOG b/CHANGELOG
index 692a22ea47..28a30c2064 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,5 +1,7 @@
 git-annex (6.20161119) UNRELEASED; urgency=medium
 
+  * enable-tor: New command, enables tor hidden service for P2P syncing.
+  * remotedaemon: Serve tor hidden service.
   * remotedaemon: Fork to background by default. Added --foreground switch
     to enable old behavior.
 
diff --git a/Command/RemoteDaemon.hs b/Command/RemoteDaemon.hs
index c68cf816a3..c174171041 100644
--- a/Command/RemoteDaemon.hs
+++ b/Command/RemoteDaemon.hs
@@ -14,7 +14,7 @@ import RemoteDaemon.Core
 import Utility.Daemon
 
 cmd :: Command
-cmd = noCommit $ dontCheck repoExists $
+cmd = noCommit $
 	command "remotedaemon" SectionMaintenance
 		"persistent communication with remotes"
 		paramNothing (run <$$> const parseDaemonOptions)
diff --git a/Remote/Helper/P2P/IO.hs b/Remote/Helper/P2P/IO.hs
index 7179adc2b5..82ba2d6f92 100644
--- a/Remote/Helper/P2P/IO.hs
+++ b/Remote/Helper/P2P/IO.hs
@@ -9,7 +9,7 @@
 
 module Remote.Helper.P2P.IO
 	( RunProto
-	, runProtoHandle
+	, runNetProtoHandle
 	) where
 
 import Remote.Helper.P2P
@@ -38,8 +38,8 @@ data S = S
 
 -- Implementation of the protocol, communicating with a peer
 -- over a Handle. No Local actions will be run.
-runProtoHandle :: MonadIO m => Handle -> Repo -> Proto a -> m a
-runProtoHandle h r = go
+runNetProtoHandle :: MonadIO m => Handle -> Repo -> Proto a -> m a
+runNetProtoHandle h r = go
   where
 	go :: RunProto
 	go (Pure a) = pure a
diff --git a/RemoteDaemon/Core.hs b/RemoteDaemon/Core.hs
index 3b3f6d98d7..446948da6d 100644
--- a/RemoteDaemon/Core.hs
+++ b/RemoteDaemon/Core.hs
@@ -45,7 +45,9 @@ runInteractive = do
 	let controller = runController ichan ochan
 	
 	-- If any thread fails, the rest will be killed.
-	void $ tryIO $ reader `concurrently` writer `concurrently` controller
+	void $ tryIO $ reader
+		`concurrently` writer
+		`concurrently` controller
 
 runNonInteractive :: IO ()
 runNonInteractive = do
@@ -59,7 +61,9 @@ runNonInteractive = do
 		void $ atomically $ readTChan ochan
 	let controller = runController ichan ochan
 	
-	void $ tryIO $ reader `concurrently` writer `concurrently` controller
+	void $ tryIO $ reader
+		`concurrently` writer
+		`concurrently` controller
 
 type RemoteMap = M.Map Git.Repo (IO (), TChan Consumed)
 
@@ -70,6 +74,7 @@ runController ichan ochan = do
 	h <- genTransportHandle
 	m <- genRemoteMap h ochan
 	startrunning m
+	mapM_ (\s -> async (s h)) remoteServers
 	go h False m
   where
 	go h paused m = do
diff --git a/RemoteDaemon/Transport.hs b/RemoteDaemon/Transport.hs
index 0e2040d1f2..6605012de3 100644
--- a/RemoteDaemon/Transport.hs
+++ b/RemoteDaemon/Transport.hs
@@ -10,6 +10,7 @@ module RemoteDaemon.Transport where
 import RemoteDaemon.Types
 import qualified RemoteDaemon.Transport.Ssh
 import qualified RemoteDaemon.Transport.GCrypt
+import qualified RemoteDaemon.Transport.Tor
 import qualified Git.GCrypt
 
 import qualified Data.Map as M
@@ -22,3 +23,6 @@ remoteTransports = M.fromList
 	[ ("ssh:", RemoteDaemon.Transport.Ssh.transport)
 	, (Git.GCrypt.urlScheme, RemoteDaemon.Transport.GCrypt.transport)
 	]
+
+remoteServers :: [TransportHandle -> IO ()]
+remoteServers = [RemoteDaemon.Transport.Tor.server]
diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs
new file mode 100644
index 0000000000..1527939b16
--- /dev/null
+++ b/RemoteDaemon/Transport/Tor.hs
@@ -0,0 +1,51 @@
+{- git-remote-daemon, tor hidden service transport
+ -
+ - Copyright 2016 Joey Hess <id@joeyh.name>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module RemoteDaemon.Transport.Tor (server) where
+
+import Common
+import RemoteDaemon.Types
+import RemoteDaemon.Common
+import Utility.Tor
+import Utility.FileMode
+import Remote.Helper.P2P
+import Remote.Helper.P2P.IO
+import Annex.UUID
+import Types.UUID
+
+import System.PosixCompat.User
+import Network.Socket
+import Control.Concurrent
+import System.Log.Logger (debugM)
+
+-- Run tor hidden service.
+server :: TransportHandle -> IO ()
+server th@(TransportHandle (LocalRepo r) _) = do
+	u <- liftAnnex th getUUID
+	uid <- getRealUserID
+	let ident = fromUUID u
+	let sock = socketFile uid ident
+	nukeFile sock
+	soc <- socket AF_UNIX Stream defaultProtocol
+	bind soc (SockAddrUnix sock)
+	-- Allow everyone to read and write to the socket; tor is probably
+	-- running as a different user. Connections have to authenticate
+	-- to do anything, so it's fine that other local users can connect.
+	modifyFileMode sock $ addModes
+		[groupReadMode, groupWriteMode, otherReadMode, otherWriteMode]
+	listen soc 2
+	debugM "remotedaemon" "tor hidden service running"
+	forever $ do
+		(conn, _) <- accept soc
+		forkIO $ do
+			debugM "remotedaemon" "handling a connection"
+			h <- socketToHandle conn ReadWriteMode
+			hSetBuffering h LineBuffering
+			hSetBinaryMode h False
+			runNetProtoHandle h r (serve u)
+			hClose h
+
diff --git a/Utility/Tor.hs b/Utility/Tor.hs
index a0a6090089..b673c71057 100644
--- a/Utility/Tor.hs
+++ b/Utility/Tor.hs
@@ -15,6 +15,7 @@ import Data.Char
 type OnionPort = Int
 type OnionAddress = String
 type OnionSocket = FilePath
+type UniqueIdent = String
 
 -- | Adds a hidden service connecting to localhost, using some kind
 -- of unique identifier.
@@ -27,7 +28,7 @@ type OnionSocket = FilePath
 -- 
 -- If there is already a hidden service for the specified unique
 -- identifier, returns its information without making any changes.
-addHiddenService :: UserID -> String -> IO (OnionAddress, OnionPort, OnionSocket)
+addHiddenService :: UserID -> UniqueIdent -> IO (OnionAddress, OnionPort, OnionSocket)
 addHiddenService uid ident = do
 	ls <- lines <$> readFile torrc
 	let portssocks = mapMaybe (parseportsock . separate isSpace) ls
@@ -39,7 +40,7 @@ addHiddenService uid ident = do
 			writeFile torrc $ unlines $
 				ls ++
 				[ ""
-				, "HiddenServiceDir " ++ hsdir
+				, "HiddenServiceDir " ++ hiddenServiceDir uid ident
 				, "HiddenServicePort " ++ show newport ++ 
 					" unix:" ++ sockfile
 				]
@@ -58,13 +59,12 @@ addHiddenService uid ident = do
 		return (p, drop 1 (dropWhile (/= ':') l))
 	parseportsock _ = Nothing
 	
-	hsdir = libDir </> "hidden_service_" ++ show uid ++ "_" ++ ident
-	sockfile = runDir uid </> ident ++ ".sock"
+	sockfile = socketFile uid ident
 
 	waithiddenservice :: Int -> OnionPort -> IO (OnionAddress, OnionPort, OnionSocket)
 	waithiddenservice 0 _ = error "tor failed to create hidden service, perhaps the tor service is not running"
 	waithiddenservice n p = do
-		v <- tryIO $ readFile (hsdir </> "hostname")
+		v <- tryIO $ readFile $ hiddenServiceHostnameFile uid ident
 		case v of
 			Right s | ".onion\n" `isSuffixOf` s -> 
 				return (takeWhile (/= '\n') s, p, sockfile)
@@ -80,3 +80,12 @@ libDir = "/var/lib/tor"
 
 runDir :: UserID -> FilePath
 runDir uid = "/var/run/user" </> show uid
+
+socketFile :: UserID -> UniqueIdent -> FilePath
+socketFile uid ident = runDir uid </> ident ++ ".sock"
+
+hiddenServiceDir :: UserID -> UniqueIdent -> FilePath
+hiddenServiceDir uid ident = libDir </> "hidden_service_" ++ show uid ++ "_" ++ ident
+
+hiddenServiceHostnameFile :: UserID -> UniqueIdent -> FilePath
+hiddenServiceHostnameFile uid ident = hiddenServiceDir uid ident </> "hostname"
diff --git a/git-annex.cabal b/git-annex.cabal
index 77c50b66e4..7a0e34b3a2 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -937,6 +937,7 @@ Executable git-annex
     RemoteDaemon.Core
     RemoteDaemon.Transport
     RemoteDaemon.Transport.GCrypt
+    RemoteDaemon.Transport.Tor
     RemoteDaemon.Transport.Ssh
     RemoteDaemon.Transport.Ssh.Types
     RemoteDaemon.Types

From 9cf9ee73f53dddc813a12be0896eed1923e70973 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sun, 20 Nov 2016 16:42:18 -0400
Subject: [PATCH 073/367] improve p2p protocol implementation

Tested it in ghci a little now.
---
 Remote/Helper/P2P.hs          |  6 +++---
 Remote/Helper/P2P/IO.hs       | 34 +++++++++++++++++++++-------------
 RemoteDaemon/Transport/Tor.hs |  2 +-
 3 files changed, 25 insertions(+), 17 deletions(-)

diff --git a/Remote/Helper/P2P.hs b/Remote/Helper/P2P.hs
index fbd6c24635..1e15195606 100644
--- a/Remote/Helper/P2P.hs
+++ b/Remote/Helper/P2P.hs
@@ -134,6 +134,7 @@ data NetF c
 	| ReceiveMessage (Message -> c)
 	| SendBytes Len L.ByteString c
 	| ReceiveBytes Len (L.ByteString -> c)
+	| CheckAuthToken UUID AuthToken (Bool -> c)
 	| Relay RelayHandle
 		(RelayData -> Net (Maybe ExitCode))
 		(ExitCode -> c)
@@ -173,7 +174,6 @@ data LocalF c
 	-- Note: The ByteString may not contain the entire remaining content
 	-- of the key. Only once the key file size == Len has the whole
 	-- content been transferred.
-	| CheckAuthToken UUID AuthToken (Bool -> c)
 	| SetPresent Key UUID c
 	| CheckContentPresent Key (Bool -> c)
 	-- ^ Checks if the whole content of the key is locally present.
@@ -203,6 +203,7 @@ runNet (ReceiveMessage _) [] = [("not enough Messages provided", Nothing)]
 runNet (ReceiveMessage next) (m:ms) = ("<", Just m):runPure (next m) ms
 runNet (SendBytes _ _ next) ms = ("> bytes", Nothing):runPure next ms
 runNet (ReceiveBytes _ next) ms = ("< bytes", Nothing):runPure (next L.empty) ms
+runNet (CheckAuthToken _ _ next) ms = runPure (next True) ms
 runNet (Relay _ _ next) ms = runPure (next ExitSuccess) ms
 runNet (RelayService _ _ next) ms = runPure (next ExitSuccess) ms
 runNet (WriteRelay _ _ next) ms = runPure next ms
@@ -211,7 +212,6 @@ runLocal :: Show r => LocalF (Proto r) -> [Message] -> [(String, Maybe Message)]
 runLocal (KeyFileSize _ next) ms = runPure (next (Len 100)) ms
 runLocal (ReadKeyFile _ _ next) ms = runPure (next L.empty) ms
 runLocal (WriteKeyFile _ _ _ _ next) ms = runPure (next True) ms
-runLocal (CheckAuthToken _ _ next) ms = runPure (next True) ms
 runLocal (SetPresent _ _ next) ms = runPure next ms
 runLocal (CheckContentPresent _ next) ms = runPure (next False) ms
 runLocal (RemoveKeyFile _ next) ms = runPure (next True) ms
@@ -298,7 +298,7 @@ serve myuuid = go Nothing
 		r <- net receiveMessage
 		case r of
 			AUTH theiruuid authtoken -> do
-				ok <- local $ checkAuthToken theiruuid authtoken
+				ok <- net $ checkAuthToken theiruuid authtoken
 				if ok
 					then do
 						net $ sendMessage (AUTH_SUCCESS myuuid)
diff --git a/Remote/Helper/P2P/IO.hs b/Remote/Helper/P2P/IO.hs
index 82ba2d6f92..6908fd68c6 100644
--- a/Remote/Helper/P2P/IO.hs
+++ b/Remote/Helper/P2P/IO.hs
@@ -14,6 +14,7 @@ module Remote.Helper.P2P.IO
 
 import Remote.Helper.P2P
 import Utility.Process
+import Types.UUID
 import Git
 import Git.Command
 import Utility.SafeCommand
@@ -33,39 +34,46 @@ type RunProto = forall a m. MonadIO m => Proto a -> m a
 
 data S = S
 	{ repo :: Repo
-	, hdl :: Handle
+	, ihdl :: Handle
+	, ohdl :: Handle
 	}
 
 -- Implementation of the protocol, communicating with a peer
 -- over a Handle. No Local actions will be run.
-runNetProtoHandle :: MonadIO m => Handle -> Repo -> Proto a -> m a
-runNetProtoHandle h r = go
+runNetProtoHandle :: MonadIO m => Handle -> Handle -> Repo -> Proto a -> m a
+runNetProtoHandle i o r = go
   where
 	go :: RunProto
 	go (Pure a) = pure a
-	go (Free (Net n)) = runNetHandle (S r h) go n
+	go (Free (Net n)) = runNetHandle (S r i o) go n
 	go (Free (Local _)) = error "local actions not allowed"
 
 runNetHandle :: MonadIO m => S -> RunProto -> NetF (Proto a) -> m a
 runNetHandle s runner f = case f of
 	SendMessage m next -> do
 		liftIO $ do
-			hPutStrLn (hdl s) (unwords (formatMessage m))
-			hFlush (hdl s)
+			hPutStrLn (ohdl s) (unwords (formatMessage m))
+			hFlush (ohdl s)
 		runner next
 	ReceiveMessage next -> do
-		l <- liftIO $ hGetLine (hdl s)
-		let m = fromMaybe (ERROR "protocol parse error")
-			(parseMessage l)
-		runner (next m)
+		l <- liftIO $ hGetLine (ihdl s)
+		case parseMessage l of
+			Just m -> runner (next m)
+			Nothing -> runner $ do
+				let e = ERROR "protocol parse error"
+				net $ sendMessage e
+				next e
 	SendBytes _len b next -> do
 		liftIO $ do
-			L.hPut (hdl s) b
-			hFlush (hdl s)
+			L.hPut (ohdl s) b
+			hFlush (ohdl s)
 		runner next
 	ReceiveBytes (Len n) next -> do
-		b <- liftIO $ L.hGet (hdl s) (fromIntegral n)
+		b <- liftIO $ L.hGet (ihdl s) (fromIntegral n)
 		runner (next b)
+	CheckAuthToken u t next -> do
+		authed <- return True -- TODO XXX FIXME really check
+		runner (next authed)
 	Relay hout callback next ->
 		runRelay runner hout callback >>= runner . next
 	RelayService service callback next ->
diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs
index 1527939b16..da30bf9448 100644
--- a/RemoteDaemon/Transport/Tor.hs
+++ b/RemoteDaemon/Transport/Tor.hs
@@ -46,6 +46,6 @@ server th@(TransportHandle (LocalRepo r) _) = do
 			h <- socketToHandle conn ReadWriteMode
 			hSetBuffering h LineBuffering
 			hSetBinaryMode h False
-			runNetProtoHandle h r (serve u)
+			runNetProtoHandle h h r (serve u)
 			hClose h
 

From 3c1c3b96fa6b7ea68b69b007e5bec38677fcf7c8 Mon Sep 17 00:00:00 2001
From: "scottgorlin@a32946b2aad278883c1690a0753241583a9855b9" <scottgorlin@web>
Date: Mon, 21 Nov 2016 00:49:23 +0000
Subject: [PATCH 074/367] Added a comment: Coldline

---
 .../comment_8_1b4eb7e0f44865cd5ff0f8ef507d99c1._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/tips/using_Google_Cloud_Storage/comment_8_1b4eb7e0f44865cd5ff0f8ef507d99c1._comment

diff --git a/doc/tips/using_Google_Cloud_Storage/comment_8_1b4eb7e0f44865cd5ff0f8ef507d99c1._comment b/doc/tips/using_Google_Cloud_Storage/comment_8_1b4eb7e0f44865cd5ff0f8ef507d99c1._comment
new file mode 100644
index 0000000000..1a71f7726a
--- /dev/null
+++ b/doc/tips/using_Google_Cloud_Storage/comment_8_1b4eb7e0f44865cd5ff0f8ef507d99c1._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="scottgorlin@a32946b2aad278883c1690a0753241583a9855b9"
+ nickname="scottgorlin"
+ avatar="http://cdn.libravatar.org/avatar/2dd1fc8add62bbf4ffefac081b322563"
+ subject="Coldline"
+ date="2016-11-21T00:49:23Z"
+ content="""
+Wanted to add that \"storageclass=COLDLINE\" appears to work seamlessly, both from my mac and arm NAS.  As far as I can tell, this appears to be a no-brainer vs glacier - builtin git annex client, simpler/cheaper billing, and no 4 hour delay!
+"""]]

From 6e6d1a8c15b856b92c90853b78791b7a8b08eee5 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 21 Nov 2016 11:30:53 -0400
Subject: [PATCH 075/367] addurl: Fix bug in checking annex.largefiles
 expressions using largerthan, mimetype, and smallerthan; the first two always
 failed to match, and the latter always matched.

---
 CHANGELOG                                     |  8 ++++++++
 Command/AddUrl.hs                             |  2 +-
 ..._annex_into_git_in___39__addurl__39__.mdwn |  2 +-
 ..._d598317883753baf02175a3bf866e08a._comment | 20 +++++++++++++++++++
 4 files changed, 30 insertions(+), 2 deletions(-)
 create mode 100644 doc/bugs/adds_file_destined_for_annex_into_git_in___39__addurl__39__/comment_1_d598317883753baf02175a3bf866e08a._comment

diff --git a/CHANGELOG b/CHANGELOG
index 3777e6d5ad..de1a16fb7d 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,11 @@
+git-annex (6.20161119) UNRELEASED; urgency=medium
+
+  * addurl: Fix bug in checking annex.largefiles expressions using
+    largerthan, mimetype, and smallerthan; the first two always failed
+    to match, and the latter always matched.
+
+ -- Joey Hess <id@joeyh.name>  Mon, 21 Nov 2016 11:27:50 -0400
+
 git-annex (6.20161118) unstable; urgency=medium
 
   * git-annex.cabal: Loosen bounds on persistent to allow 2.5, which
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index e32ceb5684..9b6ac28ea9 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -341,7 +341,7 @@ cleanup u url file key mtmp = case mtmp of
 	Nothing -> go
 	Just tmp -> do
 		largematcher <- largeFilesMatcher
-		ifM (checkFileMatcher largematcher file)
+		ifM (checkFileMatcher largematcher tmp)
 			( go
 			, do
 				liftIO $ renameFile tmp file
diff --git a/doc/bugs/adds_file_destined_for_annex_into_git_in___39__addurl__39__.mdwn b/doc/bugs/adds_file_destined_for_annex_into_git_in___39__addurl__39__.mdwn
index 5b9c76e693..7e900facf1 100644
--- a/doc/bugs/adds_file_destined_for_annex_into_git_in___39__addurl__39__.mdwn
+++ b/doc/bugs/adds_file_destined_for_annex_into_git_in___39__addurl__39__.mdwn
@@ -37,4 +37,4 @@ cached/staged changes:
 """]]
 
 
-
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/adds_file_destined_for_annex_into_git_in___39__addurl__39__/comment_1_d598317883753baf02175a3bf866e08a._comment b/doc/bugs/adds_file_destined_for_annex_into_git_in___39__addurl__39__/comment_1_d598317883753baf02175a3bf866e08a._comment
new file mode 100644
index 0000000000..e03e574f39
--- /dev/null
+++ b/doc/bugs/adds_file_destined_for_annex_into_git_in___39__addurl__39__/comment_1_d598317883753baf02175a3bf866e08a._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-11-21T15:12:54Z"
+ content="""
+It's sufficient to have "* annex.largefiles=(largerthan=100kb)"
+in .gitattributes.
+
+Even "* annex.largefiles=(largerthan=0kb)" will reproduce it.
+
+Ok, I see why.. It's running the largefile matcher on the destination file
+before it renames the temp file to it!
+
+Seems to have been broken this way ever since addurl got largefiles
+support. Testing didn't catch it because it only affects largefiles
+expressions that need to examine the file.
+
+Fixed in git. Audited other checkFileMatcher calls for this problem;
+the rest are ok.
+"""]]

From d98f071319c6f81b49d4624fb621c8a63a2e1191 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 21 Nov 2016 12:30:15 -0400
Subject: [PATCH 076/367] todo

---
 doc/todo/renameremote.mdwn | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 doc/todo/renameremote.mdwn

diff --git a/doc/todo/renameremote.mdwn b/doc/todo/renameremote.mdwn
new file mode 100644
index 0000000000..3a92bf5070
--- /dev/null
+++ b/doc/todo/renameremote.mdwn
@@ -0,0 +1,24 @@
+Sometimes a name has been used for a special remote, and you want to change
+the name. A common reason is that the special remote has become dead, and
+you want to reuse the name for a new special remote. 
+
+Initremote prevents reusing a name when the old one exists, even if the old
+one is dead. And that makes sense in general, because a dead remote can
+come back sometimes, and that would leave the repo with two special remotes
+with the same name, and so enableremote would need to be run with a uuid
+instead of a name to specify which one to enable, which is not a desirable
+state of affairs.
+
+So, add `git annex renameremote oldname newname`. This could also do a `git
+remote rename`, or equivilant. (`git remote rename` gets confused by special
+remotes not having a fetch url and fails; this can be worked around by
+manually renaming the stanza in git config.)
+
+Implementing that would need a way to remove the old name from remote.log.
+We can't remove lines from union merged files, but what we could do is
+add a new line like:
+
+	- name=oldname timestamp=<latest>
+
+And in parsing remote.log, if the UUID is "-", don't include the
+remote with that name in the the resulting map.

From 28e61898b8bf5ccc6cefc376c84e619ad3bc8a83 Mon Sep 17 00:00:00 2001
From: "https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
 <https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4@web>
Date: Mon, 21 Nov 2016 16:36:34 +0000
Subject: [PATCH 077/367] added meta field for myself

---
 ...s_file_destined_for_annex_into_git_in___39__addurl__39__.mdwn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/bugs/adds_file_destined_for_annex_into_git_in___39__addurl__39__.mdwn b/doc/bugs/adds_file_destined_for_annex_into_git_in___39__addurl__39__.mdwn
index 7e900facf1..a5cdfd6d6d 100644
--- a/doc/bugs/adds_file_destined_for_annex_into_git_in___39__addurl__39__.mdwn
+++ b/doc/bugs/adds_file_destined_for_annex_into_git_in___39__addurl__39__.mdwn
@@ -36,5 +36,6 @@ cached/staged changes:
 
 """]]
 
+[[!meta author=yoh]]
 
 > [[fixed|done]] --[[Joey]]

From 070fb9e624636ceabcff0a38007936130cdfa40f Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 21 Nov 2016 17:27:38 -0400
Subject: [PATCH 078/367] Added git-remote-tor-annex, which allows git pull and
 push to the tor hidden service.

Almost working, but there's a bug in the relaying.

Also, made tor hidden service setup pick a random port, to make it harder
to port scan.

This commit was sponsored by Boyd Stephen Smith Jr. on Patreon.
---
 Build/Mans.hs                 |  7 ++--
 CHANGELOG                     |  2 ++
 CmdLine/GitRemoteTorAnnex.hs  | 62 +++++++++++++++++++++++++++++++++
 Command/EnableTor.hs          |  8 ++---
 Makefile                      |  3 ++
 Remote/Helper/P2P.hs          | 11 ++++--
 Remote/Helper/P2P/IO.hs       | 64 ++++++++++++++++++++++-------------
 Remote/Helper/Tor.hs          | 34 +++++++++++++++++++
 RemoteDaemon/Transport/Tor.hs |  6 ++--
 Setup.hs                      |  8 +++--
 Types/Creds.hs                |  2 +-
 Utility/Tor.hs                | 40 ++++++++++++++++++----
 debian/control                |  1 +
 doc/git-annex-enable-tor.mdwn |  2 +-
 doc/git-remote-tor-annex.mdwn | 36 ++++++++++++++++++++
 git-annex.cabal               |  7 +++-
 git-annex.hs                  | 22 +++++-------
 17 files changed, 254 insertions(+), 61 deletions(-)
 create mode 100644 CmdLine/GitRemoteTorAnnex.hs
 create mode 100644 Remote/Helper/Tor.hs
 create mode 100644 doc/git-remote-tor-annex.mdwn

diff --git a/Build/Mans.hs b/Build/Mans.hs
index cf86d983db..2ea9b4197c 100644
--- a/Build/Mans.hs
+++ b/Build/Mans.hs
@@ -50,8 +50,11 @@ buildMans = do
 			else return (Just dest)
 
 isManSrc :: FilePath -> Bool
-isManSrc s = "git-annex" `isPrefixOf` (takeFileName s)
-	&& takeExtension s == ".mdwn"
+isManSrc s
+	| not (takeExtension s == ".mdwn") = False
+	| otherwise = "git-annex" `isPrefixOf` f || "git-remote-" `isPrefixOf` f
+  where
+	f = takeFileName s
 
 srcToDest :: FilePath -> FilePath
 srcToDest s = "man" </> progName s ++ ".1"
diff --git a/CHANGELOG b/CHANGELOG
index 28a30c2064..cc20ec7511 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -2,6 +2,8 @@ git-annex (6.20161119) UNRELEASED; urgency=medium
 
   * enable-tor: New command, enables tor hidden service for P2P syncing.
   * remotedaemon: Serve tor hidden service.
+  * Added git-remote-tor-annex, which allows git pull and push to the tor
+    hidden service.
   * remotedaemon: Fork to background by default. Added --foreground switch
     to enable old behavior.
 
diff --git a/CmdLine/GitRemoteTorAnnex.hs b/CmdLine/GitRemoteTorAnnex.hs
new file mode 100644
index 0000000000..bc001f42fe
--- /dev/null
+++ b/CmdLine/GitRemoteTorAnnex.hs
@@ -0,0 +1,62 @@
+{- git-remote-tor-annex program
+ -
+ - Copyright 2016 Joey Hess <id@joeyh.name>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module CmdLine.GitRemoteTorAnnex where
+
+import Common
+import qualified Annex
+import qualified Git.CurrentRepo
+import Remote.Helper.P2P
+import Remote.Helper.P2P.IO
+import Remote.Helper.Tor
+import Utility.Tor
+import Annex.UUID
+
+run :: [String] -> IO ()
+run (_remotename:address:[]) = forever $ do
+	-- gitremote-helpers protocol
+	l <- getLine
+	case l of
+		"capabilities" -> do
+			putStrLn "connect"
+			putStrLn ""
+		"connect git-upload-pack" -> go UploadPack
+		"connect git-receive-pack" -> go ReceivePack
+		_ -> error $ "git-remote-helpers protocol error at " ++ show l
+  where
+	(onionaddress, onionport)
+		| '/' `elem` address = parseAddressPort $
+			reverse $ takeWhile (/= '/') $ reverse address
+		| otherwise = parseAddressPort address
+	go service = do
+		putStrLn ""
+		hFlush stdout
+		connectService onionaddress onionport service >>= exitWith
+run (_remotename:[]) = giveup "remote address not configured"
+run _ = giveup "expected remote name and address parameters"
+
+parseAddressPort :: String -> (OnionAddress, OnionPort)
+parseAddressPort s = 
+	let (a, sp) = separate (== ':') s
+	in case readish sp of
+		Nothing -> giveup "onion address must include port number"
+		Just p -> (OnionAddress a, p)
+
+connectService :: OnionAddress -> OnionPort -> Service -> IO ExitCode
+connectService address port service = do
+	state <- Annex.new =<< Git.CurrentRepo.get
+	Annex.eval state $ do
+		authtoken <- fromMaybe nullAuthToken
+			<$> getTorAuthToken address
+		myuuid <- getUUID
+		g <- Annex.gitRepo
+		h <- liftIO $ torHandle =<< connectHiddenService address port
+		runNetProtoHandle h h g $ do
+			v <- auth myuuid authtoken
+			case v of
+				Just _theiruuid -> connect service stdin stdout
+				Nothing -> giveup $ "authentication failed, perhaps you need to set " ++ torAuthTokenEnv
diff --git a/Command/EnableTor.hs b/Command/EnableTor.hs
index 369ea75093..c581fa1d4b 100644
--- a/Command/EnableTor.hs
+++ b/Command/EnableTor.hs
@@ -24,11 +24,11 @@ start :: CmdParams -> CommandStart
 start (suserid:uuid:[]) = case readish suserid of
 	Nothing -> error "Bad userid"
 	Just userid -> do
-		(onionaddr, onionport, onionsocket) <- liftIO $
+		(OnionAddress onionaddr, onionport) <- liftIO $
 			addHiddenService userid uuid
-		liftIO $ putStrLn $
+		liftIO $ putStrLn $ 
+			"tor-annex::" ++
 			onionaddr ++ ":" ++ 
-			show onionport ++ " " ++
-			show onionsocket
+			show onionport ++ " "
 		stop
 start _ = error "Bad params"
diff --git a/Makefile b/Makefile
index e05546c526..56e725db2a 100644
--- a/Makefile
+++ b/Makefile
@@ -55,6 +55,7 @@ install-bins: build
 	install -d $(DESTDIR)$(PREFIX)/bin
 	install git-annex $(DESTDIR)$(PREFIX)/bin
 	ln -sf git-annex $(DESTDIR)$(PREFIX)/bin/git-annex-shell
+	ln -sf git-annex $(DESTDIR)$(PREFIX)/bin/git-remote-tor-annex
 
 install-misc: Build/InstallDesktopFile
 	./Build/InstallDesktopFile $(PREFIX)/bin/git-annex || true
@@ -133,6 +134,7 @@ linuxstandalone-nobuild: Build/Standalone Build/LinuxMkLibs
 	cp git-annex "$(LINUXSTANDALONE_DEST)/bin/"
 	strip "$(LINUXSTANDALONE_DEST)/bin/git-annex"
 	ln -sf git-annex "$(LINUXSTANDALONE_DEST)/bin/git-annex-shell"
+	ln -sf git-annex "$(LINUXSTANDALONE_DEST)/bin/git-remote-tor-annex"
 	zcat standalone/licences.gz > $(LINUXSTANDALONE_DEST)/LICENSE
 	cp doc/logo_16x16.png doc/logo.svg $(LINUXSTANDALONE_DEST)
 	cp standalone/trustedkeys.gpg $(LINUXSTANDALONE_DEST)
@@ -194,6 +196,7 @@ osxapp: Build/Standalone Build/OSXMkLibs
 	cp git-annex "$(OSXAPP_BASE)"
 	strip "$(OSXAPP_BASE)/git-annex"
 	ln -sf git-annex "$(OSXAPP_BASE)/git-annex-shell"
+	ln -sf git-annex "$(OSXAPP_BASE)/git-remote-tor-annex"
 	gzcat standalone/licences.gz > $(OSXAPP_BASE)/LICENSE
 	cp $(OSXAPP_BASE)/LICENSE tmp/build-dmg/LICENSE.txt
 	cp standalone/trustedkeys.gpg $(OSXAPP_DEST)/Contents/MacOS
diff --git a/Remote/Helper/P2P.hs b/Remote/Helper/P2P.hs
index 1e15195606..7e49968ee2 100644
--- a/Remote/Helper/P2P.hs
+++ b/Remote/Helper/P2P.hs
@@ -26,6 +26,12 @@ import qualified Data.ByteString.Lazy as L
 newtype AuthToken = AuthToken String
 	deriving (Show)
 
+mkAuthToken :: String -> Maybe AuthToken
+mkAuthToken = fmap AuthToken . headMaybe . lines
+
+nullAuthToken :: AuthToken
+nullAuthToken = AuthToken ""
+
 newtype Offset = Offset Integer
 	deriving (Show)
 
@@ -157,6 +163,7 @@ type Net = Free NetF
 data RelayData
 	= RelayData L.ByteString
 	| RelayMessage Message
+	deriving (Show)
 
 newtype RelayHandle = RelayHandle Handle
 
@@ -400,8 +407,8 @@ relayCallback hout (RelayMessage (DATA len)) = do
 	return Nothing
 relayCallback _ (RelayMessage (CONNECTDONE exitcode)) =
 	return (Just exitcode)
-relayCallback _ (RelayMessage _) = do
-	sendMessage (ERROR "expected DATA or CONNECTDONE")
+relayCallback _ (RelayMessage m) = do
+	sendMessage $ ERROR $ "expected DATA or CONNECTDONE not " ++ unwords (Proto.formatMessage m)
 	return (Just (ExitFailure 1))
 relayCallback _ (RelayData b) = do
 	let len = Len $ fromIntegral $ L.length b
diff --git a/Remote/Helper/P2P/IO.hs b/Remote/Helper/P2P/IO.hs
index 6908fd68c6..c6a80cdbff 100644
--- a/Remote/Helper/P2P/IO.hs
+++ b/Remote/Helper/P2P/IO.hs
@@ -19,6 +19,7 @@ import Git
 import Git.Command
 import Utility.SafeCommand
 import Utility.SimpleProtocol
+import Utility.Exception
 
 import Control.Monad
 import Control.Monad.Free
@@ -30,7 +31,7 @@ import Control.Concurrent
 import qualified Data.ByteString as B
 import qualified Data.ByteString.Lazy as L
 
-type RunProto = forall a m. MonadIO m => Proto a -> m a
+type RunProto = forall a m. (MonadIO m, MonadMask m) => Proto a -> m a
 
 data S = S
 	{ repo :: Repo
@@ -40,7 +41,7 @@ data S = S
 
 -- Implementation of the protocol, communicating with a peer
 -- over a Handle. No Local actions will be run.
-runNetProtoHandle :: MonadIO m => Handle -> Handle -> Repo -> Proto a -> m a
+runNetProtoHandle :: (MonadIO m, MonadMask m) => Handle -> Handle -> Repo -> Proto a -> m a
 runNetProtoHandle i o r = go
   where
 	go :: RunProto
@@ -48,7 +49,7 @@ runNetProtoHandle i o r = go
 	go (Free (Net n)) = runNetHandle (S r i o) go n
 	go (Free (Local _)) = error "local actions not allowed"
 
-runNetHandle :: MonadIO m => S -> RunProto -> NetF (Proto a) -> m a
+runNetHandle :: (MonadIO m, MonadMask m) => S -> RunProto -> NetF (Proto a) -> m a
 runNetHandle s runner f = case f of
 	SendMessage m next -> do
 		liftIO $ do
@@ -57,10 +58,11 @@ runNetHandle s runner f = case f of
 		runner next
 	ReceiveMessage next -> do
 		l <- liftIO $ hGetLine (ihdl s)
+		-- liftIO $ hPutStrLn stderr ("< " ++ show l)
 		case parseMessage l of
 			Just m -> runner (next m)
 			Nothing -> runner $ do
-				let e = ERROR "protocol parse error"
+				let e = ERROR $ "protocol parse error: " ++ show l
 				net $ sendMessage e
 				next e
 	SendBytes _len b next -> do
@@ -70,6 +72,7 @@ runNetHandle s runner f = case f of
 		runner next
 	ReceiveBytes (Len n) next -> do
 		b <- liftIO $ L.hGet (ihdl s) (fromIntegral n)
+		--liftIO $ hPutStrLn stderr $ "!!!" ++ show (L.length b)
 		runner (next b)
 	CheckAuthToken u t next -> do
 		authed <- return True -- TODO XXX FIXME really check
@@ -80,7 +83,8 @@ runNetHandle s runner f = case f of
 		runRelayService s runner service callback >>= runner . next
 	WriteRelay (RelayHandle h) b next -> do
 		liftIO $ do
-			L.hPut h b
+			-- L.hPut h b
+			hPutStrLn h (show ("relay got:", b, L.length b))
 			hFlush h
 		runner next
 
@@ -112,43 +116,57 @@ runRelay runner (RelayHandle hout) callback = do
 
 	drain v = do
 		d <- takeMVar v
+		liftIO $ hPutStrLn stderr (show d)
 		r <- runner $ net $ callback d
 		case r of
 			Nothing -> drain v
 			Just exitcode -> return exitcode
 
 runRelayService
-	:: MonadIO m
+	:: (MonadIO m, MonadMask m)
 	=> S
 	-> RunProto
 	-> Service
 	-> (RelayHandle -> RelayData -> Net (Maybe ExitCode))
 	-> m ExitCode
-runRelayService s runner service callback = do
-	v <- liftIO newEmptyMVar
-	(Just hin, Just hout, _, pid) <- liftIO $ createProcess serviceproc
-		{ std_out = CreatePipe
-		, std_in = CreatePipe
-		}
-	_ <- liftIO $ forkIO $ readout v hout
-	feeder <- liftIO $ forkIO $ feedin v
-	_ <- liftIO $ forkIO $ putMVar v . Left =<< waitForProcess pid
-	exitcode <- liftIO $ drain v hin
-	liftIO $ killThread feeder
-	return exitcode
+runRelayService s runner service callback = bracket setup cleanup go
   where
 	cmd = case service of
 		UploadPack -> "upload-pack"
 		ReceivePack -> "receive-pack"
-	serviceproc = gitCreateProcess [Param cmd, File (repoPath (repo s))] (repo s)
+	
+	serviceproc = gitCreateProcess
+		[ Param cmd
+		, File (repoPath (repo s))
+		] (repo s)
+
+	setup = do
+		v <- liftIO newEmptyMVar
+		(Just hin, Just hout, _, pid) <- liftIO $ 
+			createProcess serviceproc
+				{ std_out = CreatePipe
+				, std_in = CreatePipe
+				}
+		feeder <- liftIO $ forkIO $ feedin v
+		return (v, feeder, hin, hout, pid)
+
+	cleanup (_, feeder, hin, hout, pid) = liftIO $ do
+		hClose hin
+		hClose hout
+		liftIO $ killThread feeder
+		void $ waitForProcess pid
+
+	go (v, _, hin, hout, pid) = do
+		_ <- liftIO $ forkIO $ readout v hout
+		_ <- liftIO $ forkIO $ putMVar v . Left =<< waitForProcess pid
+		liftIO $ drain v hin
 
 	drain v hin = do
 		d <- takeMVar v
 		case d of
-			Left exitcode -> do
-				hClose hin
-				return exitcode
+			Left exitcode -> return exitcode
 			Right relaydata -> do
+				liftIO $ hPutStrLn stderr ("> " ++ show relaydata)
 				_ <- runner $ net $
 					callback (RelayHandle hin) relaydata
 				drain v hin
@@ -156,7 +174,7 @@ runRelayService s runner service callback = do
 	readout v hout = do
 		b <- B.hGetSome hout 65536
 		if B.null b
-			then hClose hout
+			then return ()
 			else do
 				putMVar v $ Right $ 
 					RelayData (L.fromChunks [b])
diff --git a/Remote/Helper/Tor.hs b/Remote/Helper/Tor.hs
new file mode 100644
index 0000000000..e910833621
--- /dev/null
+++ b/Remote/Helper/Tor.hs
@@ -0,0 +1,34 @@
+{- Helpers for tor remotes.
+ -
+ - Copyright 2016 Joey Hess <id@joeyh.name>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Remote.Helper.Tor where
+
+import Annex.Common
+import Remote.Helper.P2P (mkAuthToken, AuthToken)
+import Creds
+import Utility.Tor
+import Utility.Env
+
+import Network.Socket
+
+getTorAuthToken :: OnionAddress -> Annex (Maybe AuthToken)
+getTorAuthToken (OnionAddress onionaddress) =
+	maybe Nothing mkAuthToken <$> getM id 
+		[ liftIO $ getEnv torAuthTokenEnv
+		, readCacheCreds onionaddress
+		]
+
+torAuthTokenEnv :: String
+torAuthTokenEnv = "GIT_ANNEX_TOR_AUTHTOKEN"
+
+torHandle :: Socket -> IO Handle
+torHandle s = do
+	h <- socketToHandle s ReadWriteMode
+	hSetBuffering h LineBuffering
+	hSetBinaryMode h False
+	fileEncoding h
+	return h
diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs
index da30bf9448..e0922a7660 100644
--- a/RemoteDaemon/Transport/Tor.hs
+++ b/RemoteDaemon/Transport/Tor.hs
@@ -12,6 +12,7 @@ import RemoteDaemon.Types
 import RemoteDaemon.Common
 import Utility.Tor
 import Utility.FileMode
+import Remote.Helper.Tor
 import Remote.Helper.P2P
 import Remote.Helper.P2P.IO
 import Annex.UUID
@@ -43,9 +44,6 @@ server th@(TransportHandle (LocalRepo r) _) = do
 		(conn, _) <- accept soc
 		forkIO $ do
 			debugM "remotedaemon" "handling a connection"
-			h <- socketToHandle conn ReadWriteMode
-			hSetBuffering h LineBuffering
-			hSetBinaryMode h False
+			h <- torHandle conn
 			runNetProtoHandle h h r (serve u)
 			hClose h
-
diff --git a/Setup.hs b/Setup.hs
index fe06a08b17..57efd86e0c 100644
--- a/Setup.hs
+++ b/Setup.hs
@@ -33,17 +33,19 @@ main = defaultMainWithHooks simpleUserHooks
 
 myPostCopy :: Args -> CopyFlags -> PackageDescription -> LocalBuildInfo -> IO ()
 myPostCopy _ flags pkg lbi = when (System.Info.os /= "mingw32") $ do
-	installGitAnnexShell dest verbosity pkg lbi
+	installGitAnnexLinks dest verbosity pkg lbi
 	installManpages      dest verbosity pkg lbi
 	installDesktopFile   dest verbosity pkg lbi
   where
 	dest      = fromFlag $ copyDest flags
 	verbosity = fromFlag $ copyVerbosity flags
 
-installGitAnnexShell :: CopyDest -> Verbosity -> PackageDescription -> LocalBuildInfo -> IO ()
-installGitAnnexShell copyDest verbosity pkg lbi =
+installGitAnnexLinks :: CopyDest -> Verbosity -> PackageDescription -> LocalBuildInfo -> IO ()
+installGitAnnexLinks copyDest verbosity pkg lbi = do
 	rawSystemExit verbosity "ln"
 		["-sf", "git-annex", dstBinDir </> "git-annex-shell"]
+	rawSystemExit verbosity "ln"
+		["-sf", "git-annex", dstBinDir </> "git-remote-tor-annex"]
   where
 	dstBinDir = bindir $ absoluteInstallDirs pkg lbi copyDest
 
diff --git a/Types/Creds.hs b/Types/Creds.hs
index ad1827bc94..6a9e1287f7 100644
--- a/Types/Creds.hs
+++ b/Types/Creds.hs
@@ -11,4 +11,4 @@ type Creds = String -- can be any data that contains credentials
 
 type CredPair = (Login, Password)
 type Login = String
-type Password = String -- todo: use securemem
+type Password = String
diff --git a/Utility/Tor.hs b/Utility/Tor.hs
index b673c71057..eedee8c6b4 100644
--- a/Utility/Tor.hs
+++ b/Utility/Tor.hs
@@ -11,32 +11,53 @@ import Common
 import Utility.ThreadScheduler
 import System.PosixCompat.Types
 import Data.Char
+import Network.Socket
+import Network.Socks5
+import qualified Data.ByteString.UTF8 as BU8
+import qualified System.Random as R
 
 type OnionPort = Int
-type OnionAddress = String
+newtype OnionAddress = OnionAddress String
 type OnionSocket = FilePath
 type UniqueIdent = String
 
+connectHiddenService :: OnionAddress -> OnionPort -> IO Socket
+connectHiddenService (OnionAddress address) port = do
+        soc <- socket AF_UNIX Stream defaultProtocol
+	connect soc (SockAddrUnix "/run/user/1000/1ecd1f64-3234-47ec-876c-47c4bd7f7407.sock")
+	return soc
+
+connectHiddenService' :: OnionAddress -> OnionPort -> IO Socket
+connectHiddenService' (OnionAddress address) port = do
+	(s, _) <- socksConnect torsockconf socksaddr
+	return s
+  where
+	torsocksport = 9050
+	torsockconf = defaultSocksConf "127.0.0.1" torsocksport
+	socksdomain = SocksAddrDomainName (BU8.fromString address)
+	socksaddr = SocksAddress socksdomain (fromIntegral port)
+
 -- | Adds a hidden service connecting to localhost, using some kind
 -- of unique identifier.
 --
 -- This will only work if run as root, and tor has to already be running.
 --
--- Picks a port number for the hidden service that is not used by any
--- other hidden service (and is >= 1024). Returns the hidden service's
+-- Picks a random high port number for the hidden service that is not
+-- used by any other hidden service. Returns the hidden service's
 -- onion address, port, and the unix socket file to use.
 -- 
 -- If there is already a hidden service for the specified unique
 -- identifier, returns its information without making any changes.
-addHiddenService :: UserID -> UniqueIdent -> IO (OnionAddress, OnionPort, OnionSocket)
+addHiddenService :: UserID -> UniqueIdent -> IO (OnionAddress, OnionPort)
 addHiddenService uid ident = do
 	ls <- lines <$> readFile torrc
 	let portssocks = mapMaybe (parseportsock . separate isSpace) ls
 	case filter (\(_, s) -> s == sockfile) portssocks of
 		((p, _s):_) -> waithiddenservice 1 p
 		_ -> do
+			highports <- R.getStdRandom highports
 			let newport = Prelude.head $
-				filter (`notElem` map fst portssocks) [1024..]
+				filter (`notElem` map fst portssocks) highports
 			writeFile torrc $ unlines $
 				ls ++
 				[ ""
@@ -61,13 +82,18 @@ addHiddenService uid ident = do
 	
 	sockfile = socketFile uid ident
 
-	waithiddenservice :: Int -> OnionPort -> IO (OnionAddress, OnionPort, OnionSocket)
+	-- An infinite random list of high ports.
+	highports g = 
+		let (g1, g2) = R.split g
+		in (R.randomRs (1025, 65534) g1, g2)
+
+	waithiddenservice :: Int -> OnionPort -> IO (OnionAddress, OnionPort)
 	waithiddenservice 0 _ = error "tor failed to create hidden service, perhaps the tor service is not running"
 	waithiddenservice n p = do
 		v <- tryIO $ readFile $ hiddenServiceHostnameFile uid ident
 		case v of
 			Right s | ".onion\n" `isSuffixOf` s -> 
-				return (takeWhile (/= '\n') s, p, sockfile)
+				return (OnionAddress (takeWhile (/= '\n') s), p)
 			_ -> do
 				threadDelaySeconds (Seconds 1)
 				waithiddenservice (n-1) p
diff --git a/debian/control b/debian/control
index 07630dfa21..3196d8fcd2 100644
--- a/debian/control
+++ b/debian/control
@@ -77,6 +77,7 @@ Build-Depends:
 	libghc-disk-free-space-dev,
 	libghc-mountpoints-dev,
 	libghc-magic-dev,
+	libghc-socks-dev,
 	lsof [linux-any],
 	ikiwiki,
 	libimage-magick-perl,
diff --git a/doc/git-annex-enable-tor.mdwn b/doc/git-annex-enable-tor.mdwn
index 5355eef8b3..ceaa4b1212 100644
--- a/doc/git-annex-enable-tor.mdwn
+++ b/doc/git-annex-enable-tor.mdwn
@@ -10,7 +10,7 @@ git annex enable-tor userid uuid
 
 This plumbing-level command enables a tor hidden service for git-annex,
 using the specified repository uuid and userid. 
-It outputs to stdout a line of the form "address.onion:onionport socketfile"
+It outputs the address of the hidden service to stdout.
 
 This command has to be run by root, since it modifies `/etc/tor/torrc`.
 
diff --git a/doc/git-remote-tor-annex.mdwn b/doc/git-remote-tor-annex.mdwn
new file mode 100644
index 0000000000..63b459ed8e
--- /dev/null
+++ b/doc/git-remote-tor-annex.mdwn
@@ -0,0 +1,36 @@
+# NAME
+
+git-remote-tor-annex - remote helper program to talk to git-annex over tor
+
+# SYNOPSIS
+
+git fetch tor-annex::address.onion:port
+
+git remote add tor tor-annex::address.onion:port
+
+# DESCRIPTION
+
+This is a git remote helper program that allows git to pull and push
+over tor(1), communicating with a tor hidden service.
+
+The tor hidden service probably requires an authtoken to use it.
+The authtoken can be provided in the environment variable
+`GIT_ANNEX_TOR_AUTHTOKEN`. Or, if there is a file in 
+`.git/annex/creds/` matching the onion address of the hidden
+service, its first line is used as the authtoken.
+
+# SEE ALSO
+
+git-remote-helpers(1)
+
+[[git-annex]](1)
+
+[[git-annex-enable-tor]](1)
+
+[[git-annex-remotedaemon]](1)
+
+# AUTHOR
+
+Joey Hess <id@joeyh.name>
+
+Warning: Automatically converted into a man page by mdwn2man. Edit with care.
diff --git a/git-annex.cabal b/git-annex.cabal
index 7a0e34b3a2..751bd4bd44 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -59,6 +59,7 @@ Extra-Source-Files:
   doc/git-annex-dropunused.mdwn
   doc/git-annex-edit.mdwn
   doc/git-annex-enableremote.mdwn
+  doc/git-annex-enable-tor.mdwn
   doc/git-annex-examinekey.mdwn
   doc/git-annex-expire.mdwn
   doc/git-annex-find.mdwn
@@ -136,6 +137,7 @@ Extra-Source-Files:
   doc/git-annex-webapp.mdwn
   doc/git-annex-whereis.mdwn
   doc/git-annex-xmppgit.mdwn
+  doc/git-remote-tor-annex.mdwn
   doc/logo.svg
   doc/logo_16x16.png
   Build/mdwn2man
@@ -365,7 +367,8 @@ Executable git-annex
    aeson,
    unordered-containers,
    feed,
-   regex-tdfa
+   regex-tdfa,
+   socks
   CC-Options: -Wall
   GHC-Options: -Wall -fno-warn-tabs
   Extensions: PackageImports
@@ -700,6 +703,7 @@ Executable git-annex
     CmdLine.GitAnnexShell.Fields
     CmdLine.GlobalSetter
     CmdLine.Option
+    CmdLine.GitRemoteTorAnnex
     CmdLine.Seek
     CmdLine.Usage
     Command
@@ -924,6 +928,7 @@ Executable git-annex
     Remote.Helper.ReadOnly
     Remote.Helper.Special
     Remote.Helper.Ssh
+    Remote.Helper.Tor
     Remote.Hook
     Remote.List
     Remote.Rsync
diff --git a/git-annex.hs b/git-annex.hs
index ca8eecd2a8..d5fab7f477 100644
--- a/git-annex.hs
+++ b/git-annex.hs
@@ -1,6 +1,6 @@
 {- git-annex main program dispatch
  -
- - Copyright 2010-2014 Joey Hess <id@joeyh.name>
+ - Copyright 2010-2016 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -13,6 +13,7 @@ import Network.Socket (withSocketsDo)
 
 import qualified CmdLine.GitAnnex
 import qualified CmdLine.GitAnnexShell
+import qualified CmdLine.GitRemoteTorAnnex
 import qualified Test
 
 #ifdef mingw32_HOST_OS
@@ -23,20 +24,15 @@ import Utility.Env
 main :: IO ()
 main = withSocketsDo $ do
 	ps <- getArgs
+#ifdef mingw32_HOST_OS
+	winEnv
+#endif
 	run ps =<< getProgName
   where
-	run ps n
-		| isshell n = CmdLine.GitAnnexShell.run ps
-		| otherwise =
-#ifdef mingw32_HOST_OS
-			do
-				winEnv
-				gitannex ps
-#else
-			gitannex ps
-#endif
-	gitannex = CmdLine.GitAnnex.run Test.optParser Test.runner
-	isshell n = takeFileName n == "git-annex-shell"
+	run ps n = case takeFileName n of
+		"git-annex-shell" -> CmdLine.GitAnnexShell.run ps
+		"git-remote-tor-annex" -> CmdLine.GitRemoteTorAnnex.run ps
+		_  -> CmdLine.GitAnnex.run Test.optParser Test.runner ps
 
 #ifdef mingw32_HOST_OS
 {- On Windows, if HOME is not set, probe it and set it.

From 4f43da1bcdc4ec1cbad6cafd74a37d81a37e0a1a Mon Sep 17 00:00:00 2001
From: binx <binx@web>
Date: Mon, 21 Nov 2016 22:16:04 +0000
Subject: [PATCH 079/367]

---
 ...ssistant_sync_with_ssh_special_remote.mdwn | 30 +++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 doc/forum/two-way_assistant_sync_with_ssh_special_remote.mdwn

diff --git a/doc/forum/two-way_assistant_sync_with_ssh_special_remote.mdwn b/doc/forum/two-way_assistant_sync_with_ssh_special_remote.mdwn
new file mode 100644
index 0000000000..749a7937a2
--- /dev/null
+++ b/doc/forum/two-way_assistant_sync_with_ssh_special_remote.mdwn
@@ -0,0 +1,30 @@
+I am attempting to set up automatic two-way synchronization between my laptop and a server via ssh by running assistant on both machines. I want to have both machines be non-bare and unlocked.
+
+On the server:
+
+    $ mkdir ~/annex
+    $ cd annex
+    $ git init
+    $ git annex init u --version=6
+    $ echo This is test file 1. >testfile1.txt
+    $ git annex add testfile1.txt
+    $ git annex sync
+    $ git annex adjust --unlock
+    $ git annex wanted . standard
+    $ git annex group . client
+
+On my laptop:
+
+    $ git clone ssh://server/home/username/annex
+    $ cd annex
+    $ git annex init ml2 --version=6
+    $ git annex sync
+    $ git annex adjust --unlock
+    $ git annex wanted . standard
+    $ git annex group . client
+
+Everything seems to work when I manually sync. When I run
+
+    $ git annex assistant
+
+on both machines, however, I only get one-way automatic synchronization. Changes on the laptop are immediately propagated to the server. But changes on the server do not show up on the laptop until I manually sync. What am I doing wrong?

From 8ffda5f39f80644027b29db30bb231d2b6f74ce1 Mon Sep 17 00:00:00 2001
From: "neocryptek@659edac901ffbc8e541a974f8f18987eeafc63bd" <neocryptek@web>
Date: Mon, 21 Nov 2016 22:26:52 +0000
Subject: [PATCH 080/367] Added a comment

---
 ..._2_6314256da98966f4c7d02aa0d6bf94ff._comment | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 doc/forum/vanilla_git_repo_as_special_remote__63__/comment_2_6314256da98966f4c7d02aa0d6bf94ff._comment

diff --git a/doc/forum/vanilla_git_repo_as_special_remote__63__/comment_2_6314256da98966f4c7d02aa0d6bf94ff._comment b/doc/forum/vanilla_git_repo_as_special_remote__63__/comment_2_6314256da98966f4c7d02aa0d6bf94ff._comment
new file mode 100644
index 0000000000..f463554031
--- /dev/null
+++ b/doc/forum/vanilla_git_repo_as_special_remote__63__/comment_2_6314256da98966f4c7d02aa0d6bf94ff._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="neocryptek@659edac901ffbc8e541a974f8f18987eeafc63bd"
+ nickname="neocryptek"
+ avatar="http://cdn.libravatar.org/avatar/d9bfdefa9b503f1ac4844a686618374e"
+ subject="comment 2"
+ date="2016-11-21T22:26:52Z"
+ content="""
+Right, though bup also requires installation on the server. I'm looking for a way to store content into a vanilla git repo (as I don't have permission to install anything custom on the server).
+
+Since I want to store the content outside of git annex, it feels like a special remote. Though ideally it would have human readable files like:
+
+* <https://git-annex.branchable.com/todo/dumb__44___unsafe__44___human-readable_backend/>
+
+But since it's git and not just a normal (single version) filesystem, it could dedupe and save previous versions. Is there an easy way to hook git up safely to the external remote protocol:
+
+* [[special_remotes/external]]
+"""]]

From 6b992f672c57ab0826e30f1b37f169d9dd4f60c3 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 21 Nov 2016 19:24:55 -0400
Subject: [PATCH 081/367] pull/push over tor working now

Still a couple bugs:

* Closing the connection to the server leaves git upload-pack /
  receive-pack running, which could be used to DOS.

* Sometimes the data is transferred, but it fails at the end, sometimes
  with:

  git-remote-tor-annex: <socket: 10>: commitBuffer: resource vanished (Broken pipe)

  Must be a race condition around shutdown.
---
 CmdLine/GitRemoteTorAnnex.hs |   9 ++-
 Remote/Helper/P2P.hs         |  68 ++++++++--------
 Remote/Helper/P2P/IO.hs      | 150 ++++++++++++++++++-----------------
 Utility/Tor.hs               |   4 +
 4 files changed, 116 insertions(+), 115 deletions(-)

diff --git a/CmdLine/GitRemoteTorAnnex.hs b/CmdLine/GitRemoteTorAnnex.hs
index bc001f42fe..77fadc63e5 100644
--- a/CmdLine/GitRemoteTorAnnex.hs
+++ b/CmdLine/GitRemoteTorAnnex.hs
@@ -21,9 +21,7 @@ run (_remotename:address:[]) = forever $ do
 	-- gitremote-helpers protocol
 	l <- getLine
 	case l of
-		"capabilities" -> do
-			putStrLn "connect"
-			putStrLn ""
+		"capabilities" -> putStrLn "connect" >> ready
 		"connect git-upload-pack" -> go UploadPack
 		"connect git-receive-pack" -> go ReceivePack
 		_ -> error $ "git-remote-helpers protocol error at " ++ show l
@@ -33,9 +31,12 @@ run (_remotename:address:[]) = forever $ do
 			reverse $ takeWhile (/= '/') $ reverse address
 		| otherwise = parseAddressPort address
 	go service = do
+		ready
+		connectService onionaddress onionport service >>= exitWith
+	ready = do
 		putStrLn ""
 		hFlush stdout
-		connectService onionaddress onionport service >>= exitWith
+		
 run (_remotename:[]) = giveup "remote address not configured"
 run _ = giveup "expected remote name and address parameters"
 
diff --git a/Remote/Helper/P2P.hs b/Remote/Helper/P2P.hs
index 7e49968ee2..0e604a50d4 100644
--- a/Remote/Helper/P2P.hs
+++ b/Remote/Helper/P2P.hs
@@ -141,30 +141,18 @@ data NetF c
 	| SendBytes Len L.ByteString c
 	| ReceiveBytes Len (L.ByteString -> c)
 	| CheckAuthToken UUID AuthToken (Bool -> c)
-	| Relay RelayHandle
-		(RelayData -> Net (Maybe ExitCode))
-		(ExitCode -> c)
-	-- ^ Waits for data to be written to the RelayHandle, and for messages
-	-- to be received from the peer, and passes the data to the
-	-- callback, continuing until it returns an ExitCode.
-	| RelayService Service
-		(RelayHandle -> RelayData -> Net (Maybe ExitCode))
-		(ExitCode -> c)
-	-- ^ Runs a service, and waits for it to output to stdout,
-	-- and for messages to be received from the peer, and passes
-	-- the data to the callback (which is also passed the service's
-	-- stdin RelayHandle), continuing uniil the service exits.
-	| WriteRelay RelayHandle L.ByteString c
-	-- ^ Write data to a relay's handle, flushing it immediately.
+	| RelayService Service c
+	-- ^ Runs a service, relays its output to the peer, and data
+	-- from the peer to it.
+	| Relay RelayHandle RelayHandle (ExitCode -> c)
+	-- ^ Reads from the first RelayHandle, and sends the data to a
+	-- peer, while at the same time accepting input from the peer
+	-- which is sent the the second RelayHandle. Continues until 
+	-- the peer sends an ExitCode.
 	deriving (Functor)
 
 type Net = Free NetF
 
-data RelayData
-	= RelayData L.ByteString
-	| RelayMessage Message
-	deriving (Show)
-
 newtype RelayHandle = RelayHandle Handle
 
 data LocalF c
@@ -212,8 +200,7 @@ runNet (SendBytes _ _ next) ms = ("> bytes", Nothing):runPure next ms
 runNet (ReceiveBytes _ next) ms = ("< bytes", Nothing):runPure (next L.empty) ms
 runNet (CheckAuthToken _ _ next) ms = runPure (next True) ms
 runNet (Relay _ _ next) ms = runPure (next ExitSuccess) ms
-runNet (RelayService _ _ next) ms = runPure (next ExitSuccess) ms
-runNet (WriteRelay _ _ next) ms = runPure next ms
+runNet (RelayService _ next) ms = runPure next ms
 
 runLocal :: Show r => LocalF (Proto r) -> [Message] -> [(String, Maybe Message)]
 runLocal (KeyFileSize _ next) ms = runPure (next (Len 100)) ms
@@ -341,9 +328,7 @@ serve myuuid = go Nothing
 		-- setPresent not called because the peer may have
 		-- requested the data but not permanatly stored it.
 		GET offset key -> void $ sendContent key offset
-		CONNECT service -> do
-			exitcode <- net $ relayService service relayCallback
-			net $ sendMessage (CONNECTDONE exitcode)
+		CONNECT service -> net $ relayService service
 		_ -> net $ sendMessage (ERROR "unexpected command")
 
 sendContent :: Key -> Offset -> Proto Bool
@@ -399,19 +384,28 @@ readKeyFileLen key (Offset offset) = do
 connect :: Service -> Handle -> Handle -> Proto ExitCode
 connect service hin hout = do
 	net $ sendMessage (CONNECT service)
-	net $ relay (RelayHandle hin) (relayCallback (RelayHandle hout))
+	net $ relay (RelayHandle hin) (RelayHandle hout)
 
-relayCallback :: RelayHandle -> RelayData -> Net (Maybe ExitCode)
-relayCallback hout (RelayMessage (DATA len)) = do
-	writeRelay hout =<< receiveBytes len
-	return Nothing
-relayCallback _ (RelayMessage (CONNECTDONE exitcode)) =
-	return (Just exitcode)
-relayCallback _ (RelayMessage m) = do
-	sendMessage $ ERROR $ "expected DATA or CONNECTDONE not " ++ unwords (Proto.formatMessage m)
-	return (Just (ExitFailure 1))
-relayCallback _ (RelayData b) = do
+data RelayData
+	= RelayToPeer L.ByteString
+	| RelayFromPeer L.ByteString
+	| RelayDone ExitCode
+	deriving (Show)
+
+relayFromPeer :: Net RelayData
+relayFromPeer = do
+	r <- receiveMessage
+	case r of
+		CONNECTDONE exitcode -> return $ RelayDone exitcode
+		DATA len -> RelayFromPeer <$> receiveBytes len
+		_ -> do
+			sendMessage $ ERROR "expected DATA or CONNECTDONE"
+			return $ RelayDone $ ExitFailure 1
+
+relayToPeer :: RelayData -> Net ()
+relayToPeer (RelayDone exitcode) = sendMessage (CONNECTDONE exitcode)
+relayToPeer (RelayToPeer b) = do
 	let len = Len $ fromIntegral $ L.length b
 	sendMessage (DATA len)
 	sendBytes len b
-	return Nothing
+relayToPeer (RelayFromPeer _) = return ()
diff --git a/Remote/Helper/P2P/IO.hs b/Remote/Helper/P2P/IO.hs
index c6a80cdbff..b3597abf94 100644
--- a/Remote/Helper/P2P/IO.hs
+++ b/Remote/Helper/P2P/IO.hs
@@ -14,7 +14,6 @@ module Remote.Helper.P2P.IO
 
 import Remote.Helper.P2P
 import Utility.Process
-import Types.UUID
 import Git
 import Git.Command
 import Utility.SafeCommand
@@ -24,10 +23,10 @@ import Utility.Exception
 import Control.Monad
 import Control.Monad.Free
 import Control.Monad.IO.Class
-import Data.Maybe
 import System.Exit (ExitCode(..))
 import System.IO
 import Control.Concurrent
+import Control.Concurrent.Async
 import qualified Data.ByteString as B
 import qualified Data.ByteString.Lazy as L
 
@@ -58,7 +57,6 @@ runNetHandle s runner f = case f of
 		runner next
 	ReceiveMessage next -> do
 		l <- liftIO $ hGetLine (ihdl s)
-		-- liftIO $ hPutStrLn stderr ("< " ++ show l)
 		case parseMessage l of
 			Just m -> runner (next m)
 			Nothing -> runner $ do
@@ -72,64 +70,43 @@ runNetHandle s runner f = case f of
 		runner next
 	ReceiveBytes (Len n) next -> do
 		b <- liftIO $ L.hGet (ihdl s) (fromIntegral n)
-		--liftIO $ hPutStrLn stderr $ "!!!" ++ show (L.length b)
 		runner (next b)
 	CheckAuthToken u t next -> do
 		authed <- return True -- TODO XXX FIXME really check
 		runner (next authed)
-	Relay hout callback next ->
-		runRelay runner hout callback >>= runner . next
-	RelayService service callback next ->
-		runRelayService s runner service callback >>= runner . next
-	WriteRelay (RelayHandle h) b next -> do
-		liftIO $ do
-			-- L.hPut h b
-			hPutStrLn h (show ("relay got:", b, L.length b))
-			hFlush h
-		runner next
+	Relay hin hout next ->
+		runRelay runner hin hout >>= runner . next
+	RelayService service next ->
+		runRelayService s runner service >> runner next
 
 runRelay
 	:: MonadIO m
 	=> RunProto
 	-> RelayHandle
-	-> (RelayData -> Net (Maybe ExitCode))
+	-> RelayHandle
 	-> m ExitCode
-runRelay runner (RelayHandle hout) callback = do
-	v <- liftIO newEmptyMVar
-	_ <- liftIO $ forkIO $ readout v
-	feeder <- liftIO $ forkIO $ feedin v
-	exitcode <- liftIO $ drain v
-	liftIO $ killThread feeder
-	return exitcode
+runRelay runner (RelayHandle hout) (RelayHandle hin) = liftIO $
+	bracket setup cleanup go
   where
-	feedin v = forever $ do
-		m <- runner $ net receiveMessage
-		putMVar v $ RelayMessage m
+	setup = do
+		v <- newEmptyMVar
+		void $ forkIO $ relayFeeder runner v
+		void $ forkIO $ relayReader v hout
+		return v
 	
-	readout v = do
-		b <- B.hGetSome hout 65536
-		if B.null b
-			then hClose hout
-			else do
-				putMVar v $ RelayData (L.fromChunks [b])
-				readout v
-
-	drain v = do
-		d <- takeMVar v
-		liftIO $ hPutStrLn stderr (show d)
-		r <- runner $ net $ callback d
-		case r of
-			Nothing -> drain v
-			Just exitcode -> return exitcode
+	cleanup _ = do
+		hClose hin
+		hClose hout
+	
+	go v = relayHelper runner v hin
 
 runRelayService
-	:: (MonadIO m, MonadMask m)
+	:: MonadIO m
 	=> S
 	-> RunProto
 	-> Service
-	-> (RelayHandle -> RelayData -> Net (Maybe ExitCode))
-	-> m ExitCode
-runRelayService s runner service callback = bracket setup cleanup go
+	-> m ()
+runRelayService s runner service = liftIO $ bracket setup cleanup go
   where
 	cmd = case service of
 		UploadPack -> "upload-pack"
@@ -141,45 +118,70 @@ runRelayService s runner service callback = bracket setup cleanup go
 		] (repo s)
 
 	setup = do
-		v <- liftIO newEmptyMVar
-		(Just hin, Just hout, _, pid) <- liftIO $ 
-			createProcess serviceproc
-				{ std_out = CreatePipe
-				, std_in = CreatePipe
-				}
-		feeder <- liftIO $ forkIO $ feedin v
-		return (v, feeder, hin, hout, pid)
+		(Just hin, Just hout, _, pid) <- createProcess serviceproc
+			{ std_out = CreatePipe
+			, std_in = CreatePipe
+			}
+		v <- newEmptyMVar
+		feeder <- async $ relayFeeder runner v
+		reader <- async $ relayReader v hout
+		waiter <- async $ waitexit v pid
+		return (v, feeder, reader, waiter, hin, hout, pid)
 
-	cleanup (_, feeder, hin, hout, pid) = liftIO $ do
+	cleanup (_, feeder, reader, waiter, hin, hout, pid) = do
+		hPutStrLn stderr "!!!!\n\nIN CLEANUP"
+		hFlush stderr
 		hClose hin
 		hClose hout
-		liftIO $ killThread feeder
+		cancel reader
+		cancel waiter
 		void $ waitForProcess pid
 
-	go (v, _, hin, hout, pid) = do
-		_ <- liftIO $ forkIO $ readout v hout
-		_ <- liftIO $ forkIO $ putMVar v . Left =<< waitForProcess pid
-		liftIO $ drain v hin
+	go (v, _, _, _, hin, _, _) = do
+		exitcode <- relayHelper runner v hin
+		runner $ net $ relayToPeer (RelayDone exitcode)
+	
+	waitexit v pid = putMVar v . RelayDone =<< waitForProcess pid
 
-	drain v hin = do
+-- Processes RelayData as it is put into the MVar.
+relayHelper :: RunProto -> MVar RelayData -> Handle -> IO ExitCode
+relayHelper runner v hin = loop
+  where
+	loop = do
 		d <- takeMVar v
 		case d of
-			Left exitcode -> return exitcode
-			Right relaydata -> do
-				liftIO $ hPutStrLn stderr ("> " ++ show relaydata)
-				_ <- runner $ net $
-					callback (RelayHandle hin) relaydata
-				drain v hin
-	
-	readout v hout = do
+			RelayFromPeer b -> do
+				L.hPut hin b
+				hFlush hin
+				loop
+			RelayToPeer b -> do
+				runner $ net $ relayToPeer (RelayToPeer b)
+				loop
+			RelayDone exitcode -> do
+				runner $ net $ relayToPeer (RelayDone exitcode)
+				return exitcode
+
+-- Takes input from the peer, and puts it into the MVar for processing.
+-- Repeats until the peer tells it it's done.
+relayFeeder :: RunProto -> MVar RelayData -> IO ()
+relayFeeder runner v = loop
+  where
+	loop = do
+		rd <- runner $ net relayFromPeer
+		putMVar v rd
+		case rd of
+			RelayDone _ -> return ()
+			_ -> loop
+
+-- Reads input from the Handle and puts it into the MVar for relaying to
+-- the peer. Continues until EOF on the Handle.
+relayReader :: MVar RelayData -> Handle -> IO ()
+relayReader v hout = loop
+  where
+	loop = do
 		b <- B.hGetSome hout 65536
 		if B.null b
 			then return ()
 			else do
-				putMVar v $ Right $ 
-					RelayData (L.fromChunks [b])
-				readout v hout
-
-	feedin v = forever $ do
-		m <- runner $ net receiveMessage
-		putMVar v $ Right $ RelayMessage m
+				putMVar v $ RelayToPeer (L.fromChunks [b])
+				loop
diff --git a/Utility/Tor.hs b/Utility/Tor.hs
index eedee8c6b4..0900fb87e7 100644
--- a/Utility/Tor.hs
+++ b/Utility/Tor.hs
@@ -17,8 +17,12 @@ import qualified Data.ByteString.UTF8 as BU8
 import qualified System.Random as R
 
 type OnionPort = Int
+
 newtype OnionAddress = OnionAddress String
+	deriving (Show)
+
 type OnionSocket = FilePath
+
 type UniqueIdent = String
 
 connectHiddenService :: OnionAddress -> OnionPort -> IO Socket

From 9c311fb5649d3b6c832f87fd13f93b1910b19f4c Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 21 Nov 2016 19:33:57 -0400
Subject: [PATCH 082/367] fix parse of CONNECTDONE

---
 Remote/Helper/P2P.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Remote/Helper/P2P.hs b/Remote/Helper/P2P.hs
index 0e604a50d4..eaa534fbeb 100644
--- a/Remote/Helper/P2P.hs
+++ b/Remote/Helper/P2P.hs
@@ -88,7 +88,7 @@ instance Proto.Receivable Message where
 	parseCommand "AUTH-SUCCESS" = Proto.parse1 AUTH_SUCCESS
 	parseCommand "AUTH-FAILURE" = Proto.parse0 AUTH_FAILURE
 	parseCommand "CONNECT" = Proto.parse1 CONNECT
-	parseCommand "CONNECTDONE" = Proto.parse1 CONNECT
+	parseCommand "CONNECTDONE" = Proto.parse1 CONNECTDONE
 	parseCommand "CHECKPRESENT" = Proto.parse1 CHECKPRESENT
 	parseCommand "LOCKCONTENT" = Proto.parse1 LOCKCONTENT
 	parseCommand "UNLOCKCONTENT" = Proto.parse0 UNLOCKCONTENT

From ae69ebfc7c70b3e523843229fd334e60fbf13405 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 21 Nov 2016 20:56:58 -0400
Subject: [PATCH 083/367] try to gather scattered writes

git upload-pack makes some uncessary writes in sequence, this tries to
gather them together to avoid needing to send multiple DATA packets when
just one will do.

In a small pull, this reduces the average number of DATA packets from
4.5 to 2.5.
---
 Remote/Helper/P2P/IO.hs | 34 ++++++++++++++++++++++++++++------
 1 file changed, 28 insertions(+), 6 deletions(-)

diff --git a/Remote/Helper/P2P/IO.hs b/Remote/Helper/P2P/IO.hs
index b3597abf94..dd0b9631db 100644
--- a/Remote/Helper/P2P/IO.hs
+++ b/Remote/Helper/P2P/IO.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-{-# LANGUAGE RankNTypes #-}
+{-# LANGUAGE RankNTypes, CPP #-}
 
 module Remote.Helper.P2P.IO
 	( RunProto
@@ -179,9 +179,31 @@ relayReader :: MVar RelayData -> Handle -> IO ()
 relayReader v hout = loop
   where
 	loop = do
-		b <- B.hGetSome hout 65536
-		if B.null b
-			then return ()
-			else do
-				putMVar v $ RelayToPeer (L.fromChunks [b])
+		bs <- getsome []
+		case bs of
+			[] -> return ()
+			_ -> do
+				putMVar v $ RelayToPeer (L.fromChunks bs)
 				loop
+	
+	-- Waiit for the first available chunk. Then, without blocking,
+	-- try to get more chunks, in case a stream of chunks is being
+	-- written in close succession. 
+	--
+	-- On Windows, hGetNonBlocking is broken, so avoid using it there.
+	getsome [] = do
+		b <- B.hGetSome hout chunk
+		if B.null b
+			then return []
+#ifndef mingw32_HOST_OS
+			else getsome [b]
+#else
+			else return [b]
+#endif
+	getsome bs = do
+		b <- B.hGetNonBlocking hout chunk
+		if B.null b
+			then return (reverse bs)
+			else getsome (b:bs)
+	
+	chunk = 65536

From 483dbcdbef1ed313ef60bdbe1834a77e9667a87d Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 21 Nov 2016 21:22:58 -0400
Subject: [PATCH 084/367] stop cleanly when there's a IO error accessing the
 Handle

All other exceptions are let through, but IO errors accessing the handle
are to be expected, so quietly ignore.
---
 CmdLine/GitRemoteTorAnnex.hs  |   5 +-
 Remote/Helper/P2P/IO.hs       | 123 ++++++++++++++++++----------------
 RemoteDaemon/Transport/Tor.hs |   2 +-
 3 files changed, 70 insertions(+), 60 deletions(-)

diff --git a/CmdLine/GitRemoteTorAnnex.hs b/CmdLine/GitRemoteTorAnnex.hs
index 77fadc63e5..f3c3a81ae7 100644
--- a/CmdLine/GitRemoteTorAnnex.hs
+++ b/CmdLine/GitRemoteTorAnnex.hs
@@ -32,7 +32,8 @@ run (_remotename:address:[]) = forever $ do
 		| otherwise = parseAddressPort address
 	go service = do
 		ready
-		connectService onionaddress onionport service >>= exitWith
+		res <- connectService onionaddress onionport service 
+		exitWith (fromMaybe (ExitFailure 1) res)
 	ready = do
 		putStrLn ""
 		hFlush stdout
@@ -47,7 +48,7 @@ parseAddressPort s =
 		Nothing -> giveup "onion address must include port number"
 		Just p -> (OnionAddress a, p)
 
-connectService :: OnionAddress -> OnionPort -> Service -> IO ExitCode
+connectService :: OnionAddress -> OnionPort -> Service -> IO (Maybe ExitCode)
 connectService address port service = do
 	state <- Annex.new =<< Git.CurrentRepo.get
 	Annex.eval state $ do
diff --git a/Remote/Helper/P2P/IO.hs b/Remote/Helper/P2P/IO.hs
index dd0b9631db..9cd2face33 100644
--- a/Remote/Helper/P2P/IO.hs
+++ b/Remote/Helper/P2P/IO.hs
@@ -30,7 +30,7 @@ import Control.Concurrent.Async
 import qualified Data.ByteString as B
 import qualified Data.ByteString.Lazy as L
 
-type RunProto = forall a m. (MonadIO m, MonadMask m) => Proto a -> m a
+type RunProto = forall a m. (MonadIO m, MonadMask m) => Proto a -> m (Maybe a)
 
 data S = S
 	{ repo :: Repo
@@ -40,58 +40,66 @@ data S = S
 
 -- Implementation of the protocol, communicating with a peer
 -- over a Handle. No Local actions will be run.
-runNetProtoHandle :: (MonadIO m, MonadMask m) => Handle -> Handle -> Repo -> Proto a -> m a
+runNetProtoHandle :: (MonadIO m, MonadMask m) => Handle -> Handle -> Repo -> Proto a -> m (Maybe a)
 runNetProtoHandle i o r = go
   where
 	go :: RunProto
-	go (Pure a) = pure a
+	go (Pure v) = pure (Just v)
 	go (Free (Net n)) = runNetHandle (S r i o) go n
-	go (Free (Local _)) = error "local actions not allowed"
+	go (Free (Local _)) = return Nothing
 
-runNetHandle :: (MonadIO m, MonadMask m) => S -> RunProto -> NetF (Proto a) -> m a
+runNetHandle :: (MonadIO m, MonadMask m) => S -> RunProto -> NetF (Proto a) -> m (Maybe a)
 runNetHandle s runner f = case f of
 	SendMessage m next -> do
-		liftIO $ do
+		v <- liftIO $ tryIO $ do
 			hPutStrLn (ohdl s) (unwords (formatMessage m))
 			hFlush (ohdl s)
-		runner next
+		case v of
+			Left _e -> return Nothing
+			Right () -> runner next
 	ReceiveMessage next -> do
-		l <- liftIO $ hGetLine (ihdl s)
-		case parseMessage l of
-			Just m -> runner (next m)
-			Nothing -> runner $ do
-				let e = ERROR $ "protocol parse error: " ++ show l
-				net $ sendMessage e
-				next e
+		v <- liftIO $ tryIO $ hGetLine (ihdl s)
+		case v of
+			Left _e -> return Nothing
+			Right l -> case parseMessage l of
+				Just m -> runner (next m)
+				Nothing -> runner $ do
+					let e = ERROR $ "protocol parse error: " ++ show l
+					net $ sendMessage e
+					next e
 	SendBytes _len b next -> do
-		liftIO $ do
+		v <- liftIO $ tryIO $ do
 			L.hPut (ohdl s) b
 			hFlush (ohdl s)
-		runner next
+		case v of
+			Left _e -> return Nothing
+			Right () -> runner next
 	ReceiveBytes (Len n) next -> do
-		b <- liftIO $ L.hGet (ihdl s) (fromIntegral n)
-		runner (next b)
+		v <- liftIO $ tryIO $ L.hGet (ihdl s) (fromIntegral n)
+		case v of
+			Left _e -> return Nothing
+			Right b -> runner (next b)
 	CheckAuthToken u t next -> do
 		authed <- return True -- TODO XXX FIXME really check
 		runner (next authed)
-	Relay hin hout next ->
-		runRelay runner hin hout >>= runner . next
-	RelayService service next ->
-		runRelayService s runner service >> runner next
+	Relay hin hout next -> do
+		v <- liftIO $ runRelay runner hin hout
+		case v of
+			Nothing -> return Nothing
+			Just exitcode -> runner (next exitcode)
+	RelayService service next -> do
+		v <- liftIO $ runRelayService s runner service
+		case v of
+			Nothing -> return Nothing
+			Just () -> runner next
 
-runRelay
-	:: MonadIO m
-	=> RunProto
-	-> RelayHandle
-	-> RelayHandle
-	-> m ExitCode
-runRelay runner (RelayHandle hout) (RelayHandle hin) = liftIO $
-	bracket setup cleanup go
+runRelay :: RunProto -> RelayHandle -> RelayHandle -> IO (Maybe ExitCode)
+runRelay runner (RelayHandle hout) (RelayHandle hin) = bracket setup cleanup go
   where
 	setup = do
 		v <- newEmptyMVar
-		void $ forkIO $ relayFeeder runner v
-		void $ forkIO $ relayReader v hout
+		void $ async $ relayFeeder runner v
+		void $ async $ relayReader v hout
 		return v
 	
 	cleanup _ = do
@@ -100,13 +108,8 @@ runRelay runner (RelayHandle hout) (RelayHandle hin) = liftIO $
 	
 	go v = relayHelper runner v hin
 
-runRelayService
-	:: MonadIO m
-	=> S
-	-> RunProto
-	-> Service
-	-> m ()
-runRelayService s runner service = liftIO $ bracket setup cleanup go
+runRelayService :: S -> RunProto -> Service -> IO (Maybe ())
+runRelayService s runner service = bracket setup cleanup go
   where
 	cmd = case service of
 		UploadPack -> "upload-pack"
@@ -123,28 +126,29 @@ runRelayService s runner service = liftIO $ bracket setup cleanup go
 			, std_in = CreatePipe
 			}
 		v <- newEmptyMVar
-		feeder <- async $ relayFeeder runner v
-		reader <- async $ relayReader v hout
+		void $ async $ relayFeeder runner v
+		void $ async $ relayReader v hout
 		waiter <- async $ waitexit v pid
-		return (v, feeder, reader, waiter, hin, hout, pid)
+		return (v, waiter, hin, hout, pid)
 
-	cleanup (_, feeder, reader, waiter, hin, hout, pid) = do
+	cleanup (_, waiter, hin, hout, pid) = do
 		hPutStrLn stderr "!!!!\n\nIN CLEANUP"
 		hFlush stderr
 		hClose hin
 		hClose hout
-		cancel reader
 		cancel waiter
 		void $ waitForProcess pid
 
-	go (v, _, _, _, hin, _, _) = do
-		exitcode <- relayHelper runner v hin
-		runner $ net $ relayToPeer (RelayDone exitcode)
+	go (v, _, hin, _, _) = do
+		r <- relayHelper runner v hin
+		case r of
+			Nothing -> return Nothing
+			Just exitcode -> runner $ net $ relayToPeer (RelayDone exitcode)
 	
 	waitexit v pid = putMVar v . RelayDone =<< waitForProcess pid
 
 -- Processes RelayData as it is put into the MVar.
-relayHelper :: RunProto -> MVar RelayData -> Handle -> IO ExitCode
+relayHelper :: RunProto -> MVar RelayData -> Handle -> IO (Maybe ExitCode)
 relayHelper runner v hin = loop
   where
 	loop = do
@@ -155,11 +159,13 @@ relayHelper runner v hin = loop
 				hFlush hin
 				loop
 			RelayToPeer b -> do
-				runner $ net $ relayToPeer (RelayToPeer b)
-				loop
+				r <- runner $ net $ relayToPeer (RelayToPeer b)
+				case r of
+					Nothing -> return Nothing
+					Just () -> loop
 			RelayDone exitcode -> do
-				runner $ net $ relayToPeer (RelayDone exitcode)
-				return exitcode
+				_ <- runner $ net $ relayToPeer (RelayDone exitcode)
+				return (Just exitcode)
 
 -- Takes input from the peer, and puts it into the MVar for processing.
 -- Repeats until the peer tells it it's done.
@@ -167,11 +173,14 @@ relayFeeder :: RunProto -> MVar RelayData -> IO ()
 relayFeeder runner v = loop
   where
 	loop = do
-		rd <- runner $ net relayFromPeer
-		putMVar v rd
-		case rd of
-			RelayDone _ -> return ()
-			_ -> loop
+		mrd <- runner $ net relayFromPeer
+		case mrd of
+			Nothing -> return ()
+			Just rd -> do
+				putMVar v rd
+				case rd of
+					RelayDone _ -> return ()
+					_ -> loop
 
 -- Reads input from the Handle and puts it into the MVar for relaying to
 -- the peer. Continues until EOF on the Handle.
diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs
index e0922a7660..8e27bc7dd3 100644
--- a/RemoteDaemon/Transport/Tor.hs
+++ b/RemoteDaemon/Transport/Tor.hs
@@ -45,5 +45,5 @@ server th@(TransportHandle (LocalRepo r) _) = do
 		forkIO $ do
 			debugM "remotedaemon" "handling a connection"
 			h <- torHandle conn
-			runNetProtoHandle h h r (serve u)
+			_ <- runNetProtoHandle h h r (serve u)
 			hClose h

From 2da338bb8d39b8c4a1c90ff346edd46bb2ca412e Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 21 Nov 2016 21:45:56 -0400
Subject: [PATCH 085/367] detect EOF on socket and cleanly shutdown the service
 process

---
 Remote/Helper/P2P/IO.hs       | 4 ++--
 RemoteDaemon/Transport/Tor.hs | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/Remote/Helper/P2P/IO.hs b/Remote/Helper/P2P/IO.hs
index 9cd2face33..76d07a11cf 100644
--- a/Remote/Helper/P2P/IO.hs
+++ b/Remote/Helper/P2P/IO.hs
@@ -168,14 +168,14 @@ relayHelper runner v hin = loop
 				return (Just exitcode)
 
 -- Takes input from the peer, and puts it into the MVar for processing.
--- Repeats until the peer tells it it's done.
+-- Repeats until the peer tells it it's done or hangs up.
 relayFeeder :: RunProto -> MVar RelayData -> IO ()
 relayFeeder runner v = loop
   where
 	loop = do
 		mrd <- runner $ net relayFromPeer
 		case mrd of
-			Nothing -> return ()
+			Nothing -> putMVar v (RelayDone (ExitFailure 1))
 			Just rd -> do
 				putMVar v rd
 				case rd of
diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs
index 8e27bc7dd3..f28a2c6fdc 100644
--- a/RemoteDaemon/Transport/Tor.hs
+++ b/RemoteDaemon/Transport/Tor.hs
@@ -47,3 +47,4 @@ server th@(TransportHandle (LocalRepo r) _) = do
 			h <- torHandle conn
 			_ <- runNetProtoHandle h h r (serve u)
 			hClose h
+			debugM "remotedaemon" "done handling a connection"

From e053f31816b7bfbc1a915cdb4d9ada5174f0c691 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 21 Nov 2016 22:03:29 -0400
Subject: [PATCH 086/367] avoid serving more than 10 tor connections at a time

Another 10 clients can be accepted and waiting their turn. After that,
start dropping connections.

This is to avoid DOS attacks..
---
 RemoteDaemon/Transport/Tor.hs | 37 ++++++++++++++++++++++++++++++-----
 1 file changed, 32 insertions(+), 5 deletions(-)

diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs
index f28a2c6fdc..f8dfede104 100644
--- a/RemoteDaemon/Transport/Tor.hs
+++ b/RemoteDaemon/Transport/Tor.hs
@@ -17,16 +17,24 @@ import Remote.Helper.P2P
 import Remote.Helper.P2P.IO
 import Annex.UUID
 import Types.UUID
+import Messages
+import Git
 
 import System.PosixCompat.User
 import Network.Socket
 import Control.Concurrent
 import System.Log.Logger (debugM)
+import Control.Concurrent.STM
 
 -- Run tor hidden service.
 server :: TransportHandle -> IO ()
 server th@(TransportHandle (LocalRepo r) _) = do
 	u <- liftAnnex th getUUID
+
+	q <- newTBQueueIO maxConnections
+	replicateM_ maxConnections $
+		forkIO $ forever $ serveClient u r q
+
 	uid <- getRealUserID
 	let ident = fromUUID u
 	let sock = socketFile uid ident
@@ -42,9 +50,28 @@ server th@(TransportHandle (LocalRepo r) _) = do
 	debugM "remotedaemon" "tor hidden service running"
 	forever $ do
 		(conn, _) <- accept soc
-		forkIO $ do
-			debugM "remotedaemon" "handling a connection"
-			h <- torHandle conn
-			_ <- runNetProtoHandle h h r (serve u)
+		h <- torHandle conn
+		ok <- atomically $ ifM (isFullTBQueue q)
+			( return False
+			, do
+				writeTBQueue q h
+				return True
+			)
+		unless ok $ do
 			hClose h
-			debugM "remotedaemon" "done handling a connection"
+			warningIO "dropped TOR connection, too busy"
+
+-- How many clients to serve at a time, maximum. This is to avoid DOS
+-- attacks.
+maxConnections :: Int
+maxConnections = 10
+
+serveClient :: UUID -> Repo -> TBQueue Handle -> IO ()
+serveClient u r q = bracket setup cleanup go
+  where
+	setup = atomically $ readTBQueue q
+	cleanup = hClose
+	go h = do
+		debugM "remotedaemon" "serving a TOR connection"
+		void $ runNetProtoHandle h h r (serve u)
+		debugM "remotedaemon" "done with TOR connection"

From 57a9484fbc9a0849881c5f1a9ac6a522d20bb9d3 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 21 Nov 2016 22:11:53 -0400
Subject: [PATCH 087/367] remove debug

---
 Remote/Helper/P2P/IO.hs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/Remote/Helper/P2P/IO.hs b/Remote/Helper/P2P/IO.hs
index 76d07a11cf..c042d0bcce 100644
--- a/Remote/Helper/P2P/IO.hs
+++ b/Remote/Helper/P2P/IO.hs
@@ -132,8 +132,6 @@ runRelayService s runner service = bracket setup cleanup go
 		return (v, waiter, hin, hout, pid)
 
 	cleanup (_, waiter, hin, hout, pid) = do
-		hPutStrLn stderr "!!!!\n\nIN CLEANUP"
-		hFlush stderr
 		hClose hin
 		hClose hout
 		cancel waiter

From 72659f3762df5f49967c419b0c3bce62e8ee51b9 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 21 Nov 2016 22:36:34 -0400
Subject: [PATCH 088/367] devblog

---
 ..._428-429__git_push_to_hiddden_service.mdwn | 31 +++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 doc/devblog/day_428-429__git_push_to_hiddden_service.mdwn

diff --git a/doc/devblog/day_428-429__git_push_to_hiddden_service.mdwn b/doc/devblog/day_428-429__git_push_to_hiddden_service.mdwn
new file mode 100644
index 0000000000..ec1bf12fd4
--- /dev/null
+++ b/doc/devblog/day_428-429__git_push_to_hiddden_service.mdwn
@@ -0,0 +1,31 @@
+The `tor` branch is coming along nicely.
+
+This weekend, I continued working on the P2P protocol, implementing
+it for network sockets, and extending it to support connecting up
+git-send-pack/git-receive-pack. 
+
+There was a bit of a detour when I split the Free monad into two separate
+ones, one for Net operations and the other for Local filesystem operations.
+
+This weekend's work was sponsored by Thomas Hochstein on Patreon.
+
+----
+
+Today, implemented a `git-remote-tor-annex` command that git will
+use for tor-annex:: urls,  and made `git annex remotedaemon`
+serve the tor hidden service.
+
+Now I have git push/pull working to the hidden service, for example:
+
+	git pull tor-annex::eeaytkuhaupbarfi.onion:47651
+
+That works very well, but does not yet check that the user is authorized
+to use the repo, beyond knowing the onion address. And currently
+it only works in git-annex repos; with some tweaks it should
+also work in plain git repos.
+
+Next, I need to teach git-annex how to access tor-annex remotes.
+And after that, an interface in the webapp for setting them up and
+connecting them together.
+
+Today's work was sponsored by Josh Taylor on Patreon.

From ae9f99f34255e494d4e0617ff10c5e35d58ca06c Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 21 Nov 2016 23:46:59 -0400
Subject: [PATCH 089/367] Relicense 5 source files that are not part of the
 webapp from AGPL to GPL.

Building w/o the webapp is not supposed to pull in any AGPLed files.

I appear to have written all the code in these files;
the only commit by anyone else is 64e844e1fefca513d143e3f74a54dfc6d526a2ff
and is a spelling fix that is not copyrightable.
---
 Assistant/Fsck.hs    | 2 +-
 Assistant/Gpg.hs     | 2 +-
 Assistant/Repair.hs  | 2 +-
 Assistant/Restart.hs | 2 +-
 Assistant/Upgrade.hs | 2 +-
 CHANGELOG            | 1 +
 6 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/Assistant/Fsck.hs b/Assistant/Fsck.hs
index 067c7d1a10..9d8848ba98 100644
--- a/Assistant/Fsck.hs
+++ b/Assistant/Fsck.hs
@@ -2,7 +2,7 @@
  -
  - Copyright 2013 Joey Hess <id@joeyh.name>
  -
- - Licensed under the GNU AGPL version 3 or higher.
+ - Licensed under the GNU GPL version 3 or higher.
  -}
 
 module Assistant.Fsck where
diff --git a/Assistant/Gpg.hs b/Assistant/Gpg.hs
index 8910e83854..34d00a3842 100644
--- a/Assistant/Gpg.hs
+++ b/Assistant/Gpg.hs
@@ -2,7 +2,7 @@
  -
  - Copyright 2013 Joey Hess <id@joeyh.name>
  -
- - Licensed under the GNU AGPL version 3 or higher.
+ - Licensed under the GNU GPL version 3 or higher.
  -}
 
 module Assistant.Gpg where
diff --git a/Assistant/Repair.hs b/Assistant/Repair.hs
index 97c9f7f94a..29bdc44f1c 100644
--- a/Assistant/Repair.hs
+++ b/Assistant/Repair.hs
@@ -2,7 +2,7 @@
  -
  - Copyright 2013 Joey Hess <id@joeyh.name>
  -
- - Licensed under the GNU AGPL version 3 or higher.
+ - Licensed under the GNU GPL version 3 or higher.
  -}
 
 {-# LANGUAGE CPP #-}
diff --git a/Assistant/Restart.hs b/Assistant/Restart.hs
index e9d152ab7f..be1b213929 100644
--- a/Assistant/Restart.hs
+++ b/Assistant/Restart.hs
@@ -2,7 +2,7 @@
  -
  - Copyright 2013 Joey Hess <id@joeyh.name>
  -
- - Licensed under the GNU AGPL version 3 or higher.
+ - Licensed under the GNU GPL version 3 or higher.
  -}
 
 {-# LANGUAGE CPP #-}
diff --git a/Assistant/Upgrade.hs b/Assistant/Upgrade.hs
index 28b838dc2b..afbb61924a 100644
--- a/Assistant/Upgrade.hs
+++ b/Assistant/Upgrade.hs
@@ -2,7 +2,7 @@
  -
  - Copyright 2013 Joey Hess <id@joeyh.name>
  -
- - Licensed under the GNU AGPL version 3 or higher.
+ - Licensed under the GNU GPL version 3 or higher.
  -}
 
 {-# LANGUAGE CPP #-}
diff --git a/CHANGELOG b/CHANGELOG
index de1a16fb7d..1e108d4a0b 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -3,6 +3,7 @@ git-annex (6.20161119) UNRELEASED; urgency=medium
   * addurl: Fix bug in checking annex.largefiles expressions using
     largerthan, mimetype, and smallerthan; the first two always failed
     to match, and the latter always matched.
+  * Relicense 5 source files that are not part of the webapp from AGPL to GPL.
 
  -- Joey Hess <id@joeyh.name>  Mon, 21 Nov 2016 11:27:50 -0400
 

From 6f11c75d39c5e1b47321cedfcb3dc794f725463a Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 22 Nov 2016 01:04:17 -0400
Subject: [PATCH 090/367] cleanup

---
 doc/git-annex-rekey.mdwn | 2 --
 1 file changed, 2 deletions(-)

diff --git a/doc/git-annex-rekey.mdwn b/doc/git-annex-rekey.mdwn
index 7dbe6ae967..4ec0b49e84 100644
--- a/doc/git-annex-rekey.mdwn
+++ b/doc/git-annex-rekey.mdwn
@@ -20,8 +20,6 @@ Multiple pairs of file and key can be given in a single command line.
   Allow rekeying of even files whose content is not currently available.
   Use with caution.
 
-# OPTIONS
-
 # SEE ALSO
 
 [[git-annex]](1)

From 48d8c175f8cc17a6cfe44037fa4e89f450fbde52 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 22 Nov 2016 01:16:18 -0400
Subject: [PATCH 091/367] avoid backtrace when rekey cntent verification fails

---
 Command/ReKey.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/ReKey.hs b/Command/ReKey.hs
index 51f9f6fe15..33734ebe79 100644
--- a/Command/ReKey.hs
+++ b/Command/ReKey.hs
@@ -44,7 +44,7 @@ perform :: FilePath -> Key -> Key -> CommandPerform
 perform file oldkey newkey = do
 	ifM (inAnnex oldkey) 
 		( unlessM (linkKey file oldkey newkey) $
-			error "failed"
+			giveup "failed"
 		, unlessM (Annex.getState Annex.force) $
 			giveup $ file ++ " is not available (use --force to override)"
 		)

From 9f179ae8b93d113b5877cade3e5a336a5cb8d0c4 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 22 Nov 2016 11:12:33 -0400
Subject: [PATCH 092/367] fix regression

The file matcher needs to be run on the destination file not the tmp
file, in order for filename matches to work properly. However, it also
needs to be able to probe the file for size and mime type.

This is a quick fix to a regression. The double rename is not pretty.
It would be good to either have a way to run the largeFileMatcher
such that it is matching on the final filename but looks at the temp
file, or to make addAnnexedFile not need the temp file in a different
location.
---
 Command/AddUrl.hs | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 9b6ac28ea9..e49d2727c7 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -340,13 +340,18 @@ cleanup :: UUID -> URLString -> FilePath -> Key -> Maybe FilePath -> Annex ()
 cleanup u url file key mtmp = case mtmp of
 	Nothing -> go
 	Just tmp -> do
+		-- Move to final location for large file check.
+		liftIO $ renameFile tmp file
 		largematcher <- largeFilesMatcher
-		ifM (checkFileMatcher largematcher tmp)
-			( go
-			, do
-				liftIO $ renameFile tmp file
-				void $ Command.Add.addSmall file
-			)
+		large <- checkFileMatcher largematcher file
+		if large
+			then do
+				-- Move back to tmp because addAnnexedFile
+				-- needs the file in a different location
+				-- than the work tree file.
+				liftIO $ renameFile file tmp
+				go
+			else void $ Command.Add.addSmall file
   where
 	go = do
 		maybeShowJSON $ JSONChunk [("key", key2file key)]

From af4d919793ac1bd3714a5a04eaa8886084dc613e Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 22 Nov 2016 14:18:34 -0400
Subject: [PATCH 093/367] unified AuthToken type between webapp and tor

---
 Assistant/Threads/WebApp.hs       |  3 +-
 Assistant/WebApp.hs               |  2 +-
 Assistant/WebApp/Notifications.hs |  1 +
 Assistant/WebApp/Types.hs         |  1 +
 CmdLine/GitRemoteTorAnnex.hs      |  3 +-
 Remote/Helper/P2P.hs              | 14 +----
 Remote/Helper/Tor.hs              | 12 ++--
 Utility/AuthToken.hs              | 99 +++++++++++++++++++++++++++++++
 Utility/WebApp.hs                 | 25 +-------
 git-annex.cabal                   |  5 +-
 10 files changed, 120 insertions(+), 45 deletions(-)
 create mode 100644 Utility/AuthToken.hs

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index f9a456f357..576feb5f01 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -39,6 +39,7 @@ import Assistant.WebApp.OtherRepos
 import Assistant.WebApp.Repair
 import Assistant.Types.ThreadedMonad
 import Utility.WebApp
+import Utility.AuthToken
 import Utility.Tmp
 import Utility.FileMode
 import Git
@@ -75,7 +76,7 @@ webAppThread assistantdata urlrenderer noannex cannotrun postfirstrun listenhost
 #endif
 	webapp <- WebApp
 		<$> pure assistantdata
-		<*> genAuthToken
+		<*> genAuthToken 512
 		<*> getreldir
 		<*> pure staticRoutes
 		<*> pure postfirstrun
diff --git a/Assistant/WebApp.hs b/Assistant/WebApp.hs
index 52608e7eef..3d0b632717 100644
--- a/Assistant/WebApp.hs
+++ b/Assistant/WebApp.hs
@@ -14,7 +14,7 @@ import Assistant.WebApp.Types
 import Assistant.Common
 import Utility.NotificationBroadcaster
 import Utility.Yesod
-import Utility.WebApp
+import Utility.AuthToken
 
 import Data.Text (Text)
 import Control.Concurrent
diff --git a/Assistant/WebApp/Notifications.hs b/Assistant/WebApp/Notifications.hs
index 8d4a86cc7b..d7d88bece2 100644
--- a/Assistant/WebApp/Notifications.hs
+++ b/Assistant/WebApp/Notifications.hs
@@ -17,6 +17,7 @@ import Assistant.Types.Buddies
 import Utility.NotificationBroadcaster
 import Utility.Yesod
 import Utility.WebApp
+import Utility.AuthToken
 
 import Data.Text (Text)
 import qualified Data.Text as T
diff --git a/Assistant/WebApp/Types.hs b/Assistant/WebApp/Types.hs
index 6b0eb9413f..a5c4ec6216 100644
--- a/Assistant/WebApp/Types.hs
+++ b/Assistant/WebApp/Types.hs
@@ -20,6 +20,7 @@ import Assistant.Ssh
 import Assistant.Pairing
 import Assistant.Types.Buddies
 import Utility.NotificationBroadcaster
+import Utility.AuthToken
 import Utility.WebApp
 import Utility.Yesod
 import Types.Transfer
diff --git a/CmdLine/GitRemoteTorAnnex.hs b/CmdLine/GitRemoteTorAnnex.hs
index f3c3a81ae7..3282cc081a 100644
--- a/CmdLine/GitRemoteTorAnnex.hs
+++ b/CmdLine/GitRemoteTorAnnex.hs
@@ -14,6 +14,7 @@ import Remote.Helper.P2P
 import Remote.Helper.P2P.IO
 import Remote.Helper.Tor
 import Utility.Tor
+import Utility.AuthToken
 import Annex.UUID
 
 run :: [String] -> IO ()
@@ -53,7 +54,7 @@ connectService address port service = do
 	state <- Annex.new =<< Git.CurrentRepo.get
 	Annex.eval state $ do
 		authtoken <- fromMaybe nullAuthToken
-			<$> getTorAuthToken address
+			<$> getTorAuthTokenFor address
 		myuuid <- getUUID
 		g <- Annex.gitRepo
 		h <- liftIO $ torHandle =<< connectHiddenService address port
diff --git a/Remote/Helper/P2P.hs b/Remote/Helper/P2P.hs
index eaa534fbeb..9d9a3847b5 100644
--- a/Remote/Helper/P2P.hs
+++ b/Remote/Helper/P2P.hs
@@ -12,6 +12,7 @@ module Remote.Helper.P2P where
 import qualified Utility.SimpleProtocol as Proto
 import Types.Key
 import Types.UUID
+import Utility.AuthToken
 import Utility.Applicative
 import Utility.PartialPrelude
 
@@ -23,15 +24,6 @@ import System.Exit (ExitCode(..))
 import System.IO
 import qualified Data.ByteString.Lazy as L
 
-newtype AuthToken = AuthToken String
-	deriving (Show)
-
-mkAuthToken :: String -> Maybe AuthToken
-mkAuthToken = fmap AuthToken . headMaybe . lines
-
-nullAuthToken :: AuthToken
-nullAuthToken = AuthToken ""
-
 newtype Offset = Offset Integer
 	deriving (Show)
 
@@ -111,10 +103,6 @@ instance Proto.Serializable Len where
 	serialize (Len n) = show n
 	deserialize = Len <$$> readish
 
-instance Proto.Serializable AuthToken where
-	serialize (AuthToken s) = s
-	deserialize = Just . AuthToken
-
 instance Proto.Serializable Service where
 	serialize UploadPack = "git-upload-pack"
 	serialize ReceivePack = "git-receive-pack"
diff --git a/Remote/Helper/Tor.hs b/Remote/Helper/Tor.hs
index e910833621..25d1920235 100644
--- a/Remote/Helper/Tor.hs
+++ b/Remote/Helper/Tor.hs
@@ -8,19 +8,23 @@
 module Remote.Helper.Tor where
 
 import Annex.Common
-import Remote.Helper.P2P (mkAuthToken, AuthToken)
+import Utility.AuthToken
 import Creds
 import Utility.Tor
 import Utility.Env
 
 import Network.Socket
+import qualified Data.Text as T
 
-getTorAuthToken :: OnionAddress -> Annex (Maybe AuthToken)
-getTorAuthToken (OnionAddress onionaddress) =
-	maybe Nothing mkAuthToken <$> getM id 
+-- Read the first line of the creds file. Environment variable overrides.
+getTorAuthTokenFor :: OnionAddress -> Annex (Maybe AuthToken)
+getTorAuthTokenFor (OnionAddress onionaddress) =
+	maybe Nothing mk <$> getM id 
 		[ liftIO $ getEnv torAuthTokenEnv
 		, readCacheCreds onionaddress
 		]
+  where
+	mk = toAuthToken . T.pack . takeWhile (/= '\n')
 
 torAuthTokenEnv :: String
 torAuthTokenEnv = "GIT_ANNEX_TOR_AUTHTOKEN"
diff --git a/Utility/AuthToken.hs b/Utility/AuthToken.hs
new file mode 100644
index 0000000000..191b4f5c92
--- /dev/null
+++ b/Utility/AuthToken.hs
@@ -0,0 +1,99 @@
+{- authentication tokens
+ -
+ - Copyright 2016 Joey Hess <id@joeyh.name>
+ -
+ - License: BSD-2-clause
+ -}
+
+module Utility.AuthToken (
+	AuthToken,
+	toAuthToken,
+	fromAuthToken,
+	nullAuthToken,
+	genAuthToken,
+	AllowedAuthTokens,
+	allowedAuthTokens,
+	isAllowedAuthToken,
+) where
+
+import qualified Utility.SimpleProtocol as Proto
+import Utility.Hash
+
+import Data.SecureMem
+import Data.Maybe
+import Data.Char
+import Data.Byteable
+import qualified Data.Text as T
+import qualified Data.Text.Encoding as TE
+import qualified Data.ByteString.Lazy as L
+import "crypto-api" Crypto.Random
+
+-- | An AuthToken is stored in secue memory, with constant time comparison.
+--
+-- It can have varying length, depending on the security needs of the
+-- application.
+--
+-- To avoid decoding issues, and presentation issues, the content
+-- of an AuthToken is limited to ASCII characters a-z, and 0-9.
+-- This is enforced by all exported AuthToken constructors.
+newtype AuthToken = AuthToken SecureMem
+	deriving (Show, Eq)
+
+allowedChar :: Char -> Bool
+allowedChar c = isAsciiUpper c || isAsciiLower c || isDigit c
+
+instance Proto.Serializable AuthToken where
+	serialize = T.unpack . fromAuthToken
+	deserialize = toAuthToken . T.pack
+
+fromAuthToken :: AuthToken -> T.Text
+fromAuthToken (AuthToken t ) = TE.decodeLatin1 (toBytes t)
+
+-- | Upper-case characters are lower-cased to make them fit in the allowed
+-- character set. This allows AuthTokens to be compared effectively
+-- case-insensitively. 
+--
+-- Returns Nothing if any disallowed characters are present.
+toAuthToken :: T.Text -> Maybe AuthToken
+toAuthToken t
+	| all allowedChar s = Just $ AuthToken $ 
+		secureMemFromByteString $ TE.encodeUtf8 $ T.pack s
+	| otherwise = Nothing
+  where
+	s = map toLower $ T.unpack t
+
+-- | The empty AuthToken, for those times when you don't want any security.
+nullAuthToken :: AuthToken
+nullAuthToken = AuthToken $ secureMemFromByteString $ TE.encodeUtf8 T.empty
+
+-- | Generates an AuthToken of a specified length. This is done by
+-- generating a random bytestring, hashing it with sha2 512, and truncating
+-- to the specified length.
+--
+-- That limits the maximum length to 128, but with 512 bytes of entropy,
+-- that should be sufficient for any application.
+genAuthToken :: Int -> IO AuthToken
+genAuthToken len = do
+	g <- newGenIO :: IO SystemRandom
+	return $
+		case genBytes 512 g of
+			Left e -> error $ "failed to generate auth token: " ++ show e
+			Right (s, _) -> fromMaybe (error "auth token encoding failed") $
+				toAuthToken $ T.pack $ take len $
+					show $ sha2_512 $ L.fromChunks [s]
+
+-- | For when several AuthTokens are allowed to be used.
+newtype AllowedAuthTokens = AllowedAuthTokens [AuthToken]
+
+allowedAuthTokens :: [AuthToken] -> AllowedAuthTokens
+allowedAuthTokens = AllowedAuthTokens
+
+-- | Note that every item in the list is checked, even if the first one
+-- is allowed, so that comparison is constant-time.
+isAllowedAuthToken :: AuthToken -> AllowedAuthTokens -> Bool
+isAllowedAuthToken t (AllowedAuthTokens l) = go False l
+  where
+	go ok [] = ok
+	go ok (i:is)
+		| t == i = go True is
+		| otherwise = go ok is
diff --git a/Utility/WebApp.hs b/Utility/WebApp.hs
index 63ca335205..a90772b107 100644
--- a/Utility/WebApp.hs
+++ b/Utility/WebApp.hs
@@ -12,7 +12,7 @@ module Utility.WebApp where
 import Common
 import Utility.Tmp
 import Utility.FileMode
-import Utility.Hash
+import Utility.AuthToken
 
 import qualified Yesod
 import qualified Network.Wai as Wai
@@ -23,7 +23,6 @@ import qualified Data.CaseInsensitive as CI
 import Network.Socket
 import "crypto-api" Crypto.Random
 import qualified Web.ClientSession as CS
-import qualified Data.ByteString.Lazy as L
 import qualified Data.ByteString as B
 import qualified Data.Text as T
 import qualified Data.Text.Encoding as TE
@@ -31,8 +30,6 @@ import Blaze.ByteString.Builder.Char.Utf8 (fromText)
 import Blaze.ByteString.Builder (Builder)
 import Control.Arrow ((***))
 import Control.Concurrent
-import Data.SecureMem
-import Data.Byteable
 #ifdef __ANDROID__
 import Data.Endian
 #endif
@@ -159,24 +156,6 @@ webAppSessionBackend _ = do
 		Just . Yesod.clientSessionBackend key . fst
 			<$> Yesod.clientSessionDateCacher timeout
 
-type AuthToken = SecureMem
-
-toAuthToken :: T.Text -> AuthToken
-toAuthToken = secureMemFromByteString . TE.encodeUtf8
-
-fromAuthToken :: AuthToken -> T.Text
-fromAuthToken = TE.decodeLatin1 . toBytes
-
-{- Generates a random sha2_512 string, encapsulated in a SecureMem,
- - suitable to be used for an authentication secret. -}
-genAuthToken :: IO AuthToken
-genAuthToken = do
-	g <- newGenIO :: IO SystemRandom
-	return $
-		case genBytes 512 g of
-			Left e -> error $ "failed to generate auth token: " ++ show e
-			Right (s, _) -> toAuthToken $ T.pack $ show $ sha2_512 $ L.fromChunks [s]
-
 {- A Yesod isAuthorized method, which checks the auth cgi parameter
  - against a token extracted from the Yesod application.
  -
@@ -193,7 +172,7 @@ checkAuthToken extractAuthToken r predicate
 		webapp <- Yesod.getYesod
 		req <- Yesod.getRequest
 		let params = Yesod.reqGetParams req
-		if (toAuthToken <$> lookup "auth" params) == Just (extractAuthToken webapp)
+		if (toAuthToken =<< lookup "auth" params) == Just (extractAuthToken webapp)
 			then return Yesod.Authorized
 			else Yesod.sendResponseStatus unauthorized401 ()
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 751bd4bd44..94d1ccf9ce 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -368,7 +368,8 @@ Executable git-annex
    unordered-containers,
    feed,
    regex-tdfa,
-   socks
+   socks,
+   securemem
   CC-Options: -Wall
   GHC-Options: -Wall -fno-warn-tabs
   Extensions: PackageImports
@@ -472,7 +473,6 @@ Executable git-annex
      clientsession,
      template-haskell,
      shakespeare (>= 2.0.0),
-     securemem,
      byteable
     CPP-Options: -DWITH_WEBAPP
 
@@ -989,6 +989,7 @@ Executable git-annex
     Upgrade.V4
     Upgrade.V5
     Utility.Applicative
+    Utility.AuthToken
     Utility.Base64
     Utility.Batch
     Utility.Bloom

From b08799893fd8b22679195d3672319299dee3e3ce Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 22 Nov 2016 14:34:49 -0400
Subject: [PATCH 094/367] reorg

---
 {Remote/Helper/P2P => P2P}/IO.hs        | 2 +-
 Remote/Helper/P2P.hs => P2P/Protocol.hs | 2 +-
 git-annex.cabal                         | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)
 rename {Remote/Helper/P2P => P2P}/IO.hs (99%)
 rename Remote/Helper/P2P.hs => P2P/Protocol.hs (99%)

diff --git a/Remote/Helper/P2P/IO.hs b/P2P/IO.hs
similarity index 99%
rename from Remote/Helper/P2P/IO.hs
rename to P2P/IO.hs
index c042d0bcce..9a1243f52b 100644
--- a/Remote/Helper/P2P/IO.hs
+++ b/P2P/IO.hs
@@ -7,7 +7,7 @@
 
 {-# LANGUAGE RankNTypes, CPP #-}
 
-module Remote.Helper.P2P.IO
+module P2P.IO
 	( RunProto
 	, runNetProtoHandle
 	) where
diff --git a/Remote/Helper/P2P.hs b/P2P/Protocol.hs
similarity index 99%
rename from Remote/Helper/P2P.hs
rename to P2P/Protocol.hs
index 9d9a3847b5..381949af1b 100644
--- a/Remote/Helper/P2P.hs
+++ b/P2P/Protocol.hs
@@ -7,7 +7,7 @@
 
 {-# LANGUAGE DeriveFunctor, TemplateHaskell, FlexibleContexts, RankNTypes #-}
 
-module Remote.Helper.P2P where
+module P2P.Protocol where
 
 import qualified Utility.SimpleProtocol as Proto
 import Types.Key
diff --git a/git-annex.cabal b/git-annex.cabal
index 94d1ccf9ce..fd8ce9ce23 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -904,6 +904,8 @@ Executable git-annex
     Messages.Internal
     Messages.JSON
     Messages.Progress
+    P2P.IO
+    P2P.Protocol
     Remote
     Remote.BitTorrent
     Remote.Bup
@@ -923,8 +925,6 @@ Executable git-annex
     Remote.Helper.Hooks
     Remote.Helper.Http
     Remote.Helper.Messages
-    Remote.Helper.P2P
-    Remote.Helper.P2P.IO
     Remote.Helper.ReadOnly
     Remote.Helper.Special
     Remote.Helper.Ssh

From 158ef45d769647e8cfb4a09aa8ac2b574cb77d18 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 22 Nov 2016 14:37:19 -0400
Subject: [PATCH 095/367] add P2P.Auth

---
 Creds.hs        |  2 +-
 P2P/Auth.hs     | 30 ++++++++++++++++++++++++++++++
 git-annex.cabal |  1 +
 3 files changed, 32 insertions(+), 1 deletion(-)
 create mode 100644 P2P/Auth.hs

diff --git a/Creds.hs b/Creds.hs
index 6be9b33916..de3cd2a063 100644
--- a/Creds.hs
+++ b/Creds.hs
@@ -156,7 +156,7 @@ readCacheCredPair storage = maybe Nothing decodeCredPair
 	<$> readCacheCreds (credPairFile storage)
 
 readCacheCreds :: FilePath -> Annex (Maybe Creds)
-readCacheCreds f = liftIO . catchMaybeIO . readFile =<< cacheCredsFile f
+readCacheCreds f = liftIO . catchMaybeIO . readFileStrict =<< cacheCredsFile f
 
 cacheCredsFile :: FilePath -> Annex FilePath
 cacheCredsFile basefile = do
diff --git a/P2P/Auth.hs b/P2P/Auth.hs
new file mode 100644
index 0000000000..5c3feb7132
--- /dev/null
+++ b/P2P/Auth.hs
@@ -0,0 +1,30 @@
+{- P2P protocol, authorization
+ -
+ - Copyright 2016 Joey Hess <id@joeyh.name>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module P2P.Auth where
+
+import Common
+import Utility.AuthToken
+
+import qualified Data.Text as T
+
+-- Use .git/annex/creds/p2p to hold AuthTokens of authorized peers.
+getAuthTokens :: Annex AllowedAuthTokens
+getAuthTokens = allowedAuthTokens <$> getAuthTokens'
+
+getAuthTokens' :: Annex [AuthTokens]
+getAuthTokens' = mapMaybe toAuthToken
+	. map T.pack
+	. lines
+	. fromMaybe []
+	<$> readCacheCreds "tor"
+
+addAuthToken :: AuthToken -> Annex ()
+addAuthToken t = do
+	ts <- getAuthTokens'
+	let d = unlines $ map (T.unpack . fromAuthToken) (t:ts)
+	writeCacheCreds d "tor"
diff --git a/git-annex.cabal b/git-annex.cabal
index fd8ce9ce23..bd8c36063f 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -904,6 +904,7 @@ Executable git-annex
     Messages.Internal
     Messages.JSON
     Messages.Progress
+    P2P.Auth
     P2P.IO
     P2P.Protocol
     Remote

From bfd19e0952451545d209c4168f7a65473bccf2c8 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 22 Nov 2016 15:17:27 -0400
Subject: [PATCH 096/367] docs for stuff not fully implemented yet

---
 doc/git-annex-enable-tor.mdwn               |   8 +-
 doc/git-annex-p2p.mdwn                      |  54 +++++++++++
 doc/tips/peer_to_peer_network_with_tor.mdwn | 101 ++++++++++++++++++++
 3 files changed, 161 insertions(+), 2 deletions(-)
 create mode 100644 doc/git-annex-p2p.mdwn
 create mode 100644 doc/tips/peer_to_peer_network_with_tor.mdwn

diff --git a/doc/git-annex-enable-tor.mdwn b/doc/git-annex-enable-tor.mdwn
index ceaa4b1212..9fb55db5f2 100644
--- a/doc/git-annex-enable-tor.mdwn
+++ b/doc/git-annex-enable-tor.mdwn
@@ -10,17 +10,21 @@ git annex enable-tor userid uuid
 
 This plumbing-level command enables a tor hidden service for git-annex,
 using the specified repository uuid and userid. 
-It outputs the address of the hidden service to stdout.
 
 This command has to be run by root, since it modifies `/etc/tor/torrc`.
 
 After this command is run, `git annex remotedaemon` can be run to serve the
-hidden service.
+tor hidden service.
+
+Use the `git-annex p2p --gen-address` command to give other users access
+to your repository via the tor hidden service.
 
 # SEE ALSO
 
 [[git-annex]](1)
 
+[[git-annex-p2p-auth]](1)
+
 [[git-annex-remotedaemon]](1)
 
 # AUTHOR
diff --git a/doc/git-annex-p2p.mdwn b/doc/git-annex-p2p.mdwn
new file mode 100644
index 0000000000..41e1967ee6
--- /dev/null
+++ b/doc/git-annex-p2p.mdwn
@@ -0,0 +1,54 @@
+# NAME
+
+git-annex p2p - manage peer-to-peer connections
+
+# SYNOPSIS
+
+git annex p2p [options]
+
+# DESCRIPTION
+
+When using git-annex with peer-to-peer communication, this manages
+connections between the peers.
+
+Currently, git-annex supports peer-to-peer communication over Tor.
+
+# OPTIONS
+
+* `--gen-address [name]`
+
+  Generates one or more addresses, which allow whoever knows them to access
+  your repository. The addresses are output on standard output, one per
+  supported P2P network.
+
+  You can re-run this command repeatedly to generate as many addresses
+  as you like.
+
+  The name is an optional parameter, the name of the person or device you
+  intend to give this address to. Providig it makes it easier to use
+  `--remove-address`
+
+* `--link-remote name address`
+
+  Link the local repository to a remote repository. This sets up a git remote
+  with the specified name. The address is one generated by `--gen-address`
+  run on the remote repository.
+
+* `--remove-address [address|name]`
+
+  If you've given out an address to someone, and don't want to accept
+  connections from them anymore, this can be used to remove it.
+
+# SEE ALSO
+
+[[git-annex]](1)
+
+[[git-annex-enable-tor]](1)
+
+[[git-annex-remotedaemon]](1)
+
+# AUTHOR
+
+Joey Hess <id@joeyh.name>
+
+Warning: Automatically converted into a man page by mdwn2man. Edit with care.
diff --git a/doc/tips/peer_to_peer_network_with_tor.mdwn b/doc/tips/peer_to_peer_network_with_tor.mdwn
new file mode 100644
index 0000000000..d7461a1e43
--- /dev/null
+++ b/doc/tips/peer_to_peer_network_with_tor.mdwn
@@ -0,0 +1,101 @@
+git-annex has recently gotten support for running as a
+[Tor](http://http://torproject.org/) hidden service. This is a great, and
+very secure way to connect repositories between computers in different
+locations, without needing any central server.
+
+## the first peer
+
+First, you need to get Tor installed and running. See
+[their website](http://http://torproject.org/), or try a command like:
+
+	sudo apt-get install tor
+
+To make git-annex use Tor, run these commands in your git-annex repository:
+
+	sudo git annex enable-tor
+	git annex remotedaemon
+	git annex p2p --gen-address
+
+The p2p-auth command will output a long address, such as:
+
+	tor-annex::eeaytkuhaupbarfi.onion:4412:7f53c5b65b8957ef626fd461ceaae8056e3dbc459ae715e4
+
+At this point, git-annex is running as a tor hidden service, but
+it will only talk to peers who know that address.
+
+## adding additional peers
+
+To add a peer, get tor installed and running on it.
+
+	sudo apt-get install tor
+
+You need a git-annex repository on the new peer. It's fine to start
+with a new empty repository:
+
+	git init annex
+	cd annex
+	git annex init
+
+And make git-annex use Tor, by running these commands in the git-annex
+repository:
+
+	sudo git annex enable-tor
+	git annex remotedaemon
+
+Now, tell the new peer about the address of the first peer:
+
+	git annex p2p --link-remote peer1 tor-annnex::eeaytkuhaupbarfi.onion:4412:7f53c5b65b8957ef626fd461ceaae8056e3dbc459ae715e4
+
+(Of course, you should paste in the address you generated earlier, 
+not the example one shown above.)
+
+Now this git-annex repository will have a remote named "peer1"
+which connects, through Tor, to the repository on the other peer.
+You can run any commands you normally would to sync with that remote:
+
+	git annex sync --content peer1
+
+You can also generate an address for this new peer, by running
+`git annex p2p --gen`, and add that address to other peers using `git annex
+p2p --link-remote`. It's often useful to link peers up in both directions,
+so peer1 is a remote of peer2 and peer2 is a remote of peer1.
+
+Any number of peers can be connected this way, within reason.
+
+## git-annex remotedaemon
+
+Notice the `git annex remotedaemon` being run in the above examples.
+That command runs the Tor hidden service so that other peers
+can connect to your repository over Tor.
+
+So, you may want to arrange for the remotedaemon to be started on boot.
+You can do that with a simple cron job:
+
+	@reboot cd myannexrepo && git annex remotedaemon
+
+If you use the git-annex assistant, and have it auto-starting on boot, it
+will take care of starting the remotedaemon for you.
+
+## onion addresses and authentication
+
+You don't need to know about this, but it might be helpful to understand
+how it works.
+
+git-annex's Tor support uses onion address as the address of a git remote.
+You can `git pull`, push, etc with those onion addresses:
+
+	git pull tor-annnex::eeaytkuhaupbarfi.onion:4412
+	git remote add peer1 tor-annnex::eeaytkuhaupbarfi.onion:4412
+
+Onion addresses are semi-public. When you add a remote, they appear in your
+`.git/config` file. So, there's a second level of authentication that
+git-annex uses to make sure that only people you want to can access your
+repository over Tor. That takes the form of a long string of numbers and
+letters, like "7f53c5b65b8957ef626fd461ceaae8056e3dbc459ae715e4".
+
+The addresses generated by `git annex peer --gen-address` 
+combine the onion address with the authentication data.
+
+When you run `git annex peer --link-remote`, it sets up a git remote using
+the onion address, and it stashes the authentication data away in a file in
+`.git/annex/creds/`

From 4d436d8ea0d7b57b4d26e5a57528e08c12c985c7 Mon Sep 17 00:00:00 2001
From: "neocryptek@659edac901ffbc8e541a974f8f18987eeafc63bd" <neocryptek@web>
Date: Tue, 22 Nov 2016 21:12:51 +0000
Subject: [PATCH 097/367]

---
 doc/forum/Odd_Hybrid_Symlinks_To_Content.mdwn | 27 +++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 doc/forum/Odd_Hybrid_Symlinks_To_Content.mdwn

diff --git a/doc/forum/Odd_Hybrid_Symlinks_To_Content.mdwn b/doc/forum/Odd_Hybrid_Symlinks_To_Content.mdwn
new file mode 100644
index 0000000000..5e3b4d8b4a
--- /dev/null
+++ b/doc/forum/Odd_Hybrid_Symlinks_To_Content.mdwn
@@ -0,0 +1,27 @@
+I've somehow managed to get my indirect repository to symlink to literal content instead of object files. 
+
+By this I mean literally the symlink is pointing at the contents of the file as the filename.
+
+So if I have a blah.txt file with this content:
+
+* First line
+* second line
+
+And I ls -al to view the symlink pointer, it shows up as this:
+
+* blah.txt -> First line?second line
+
+It literally has the contents of the file as the destination filename.
+
+I've tried a couple things I could think of to re-symlink the files, but they don't seem to do anything as they think everything is fine:
+
+* git annex indirect //returns nothing
+* git annex lock blah.txt //returns nothing
+* git annex fix blah.txt //returns nothing
+* git annex fsck //returns nothing
+
+I'm actually able to find several of these files hanging around by searching for all symlinks that don't point to something in the .git directory.
+
+Is there a way for me to replace the symlinks with correct symlinks to the objects in .git/annex? Can it even figure out which ones it was supposed to point to if the symlinks are messed up (are filenames -> content hashes stored anywhere else)?
+
+Else I might have to go do some manual rebasing and history editing to try to undo the bad commits manually. I've synced this repo to another direct repo so I'll need to figure out how to manually fix that repo too (using proxy).  From what I can tell the annex/direct/master seems to be same as master and synced/master branches? Is there an [[internals]] page for direct branches besides [[direct_mode]] so I know what should be fixed where?

From be2f7080396d10ea96c90092227561d771253848 Mon Sep 17 00:00:00 2001
From: "whlabratz@d10941ab2681deb87122fd8f6da51f3dcdb8dbe5" <whlabratz@web>
Date: Wed, 23 Nov 2016 03:26:37 +0000
Subject: [PATCH 098/367]

---
 ..._git-annex-fsck_and_git-annex-whereis.mdwn | 53 +++++++++++++++++++
 1 file changed, 53 insertions(+)
 create mode 100644 doc/bugs/Inconsistent_results_between_git-annex-fsck_and_git-annex-whereis.mdwn

diff --git a/doc/bugs/Inconsistent_results_between_git-annex-fsck_and_git-annex-whereis.mdwn b/doc/bugs/Inconsistent_results_between_git-annex-fsck_and_git-annex-whereis.mdwn
new file mode 100644
index 0000000000..c9037b575b
--- /dev/null
+++ b/doc/bugs/Inconsistent_results_between_git-annex-fsck_and_git-annex-whereis.mdwn
@@ -0,0 +1,53 @@
+### Please describe the problem.
+
+I'm seeing some inconsistent results between runs of `git annex fsck` and `git annex whereis` that I'm not able to explain. When I run `git annex fsck`, it reports a few keys that only have 1 copy, and advises me to make more copies. If I run `git annex whereis --key <key>`, git annex confirms that it only knows about 1 copy of this key. If I then use `git log --stat -S'<key>'` to find the actual file that it refers to, and run `git annex whereis <file>`, git annex report 9 copies of this file. Checking on remotes shows that these files do exist on the remote, so why does `git annex fsck` and `git annex whereis` mis-report the number of copies when querying for the key - but not for the actual filename? Additionally, `git annex find --lackingcopies 1` doesn't return any results, but should if there are actually files with not enough copies?
+
+
+### What steps will reproduce the problem?
+
+
+### What version of git-annex are you using? On what operating system?
+
+5.20151208-1build1 on Ubuntu Xenial, one remote running 5.20141024~bpo70+1 on Debian Wheezy
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+[william@hactar ~/Pictures/Photo Library]$ git annex whereis SHA256E-s1071765--dbaa7f32ee44c28d6a1f0c8095e8dfd8b4ec433b144085d5097425303a510ea9
+git-annex: SHA256E-s1071765--dbaa7f32ee44c28d6a1f0c8095e8dfd8b4ec433b144085d5097425303a510ea9 not found
+git-annex: whereis: 1 failed
+[william@hactar ~/Pictures/Photo Library]$ git annex whereis --key SHA256E-s1071765--dbaa7f32ee44c28d6a1f0c8095e8dfd8b4ec433b144085d5097425303a510ea9
+whereis SHA256E-s1071765--dbaa7f32ee44c28d6a1f0c8095e8dfd8b4ec433b144085d5097425303a510ea9 (1 copy) 
+        7691934f-2542-4103-9122-2db4e6cfc887 -- hactar [here]
+ok
+[william@hactar ~/Pictures/Photo Library]$ git annex fsck --key SHA256E-s1071765--dbaa7f32ee44c28d6a1f0c8095e8dfd8b4ec433b144085d5097425303a510ea9
+fsck SHA256E-s1071765--dbaa7f32ee44c28d6a1f0c8095e8dfd8b4ec433b144085d5097425303a510ea9 
+  Only 1 of 3 trustworthy copies exist of SHA256E-s1071765--dbaa7f32ee44c28d6a1f0c8095e8dfd8b4ec433b144085d5097425303a510ea9
+  Back it up with git-annex copy.
+failed
+(recording state in git...)
+git-annex: fsck: 1 failed
+[william@hactar ~/Pictures/Photo Library]$ git log --stat -S'SHA256E-s1071765--dbaa7f32ee44c28d6a1f0c8095e8dfd8b4ec433b144085d5097425303a510ea9'
+[william@hactar ~/Pictures/Photo Library]$ git annex whereis 2009/05/05/P1040890.JPG
+whereis 2009/05/05/P1040890.JPG (9 copies) 
+        0e825a69-1927-4f62-b731-6f3e98bba998 -- william@marvin:/media/backup/annex/photos [marvin]
+        1b728ab5-1e32-45a6-bc11-2a4bfdc9d6ab -- backup1
+        5c0caa42-b489-467b-a612-9590fa9d5a94 -- backup2
+        7691934f-2542-4103-9122-2db4e6cfc887 -- hactar [here]
+        894b2216-72e0-40e1-8765-1386e1e9e4b4 -- backup3
+        96f19fa8-d385-4e8b-b000-61ee15993a70 -- backup3
+        a862b121-d794-4af4-bb56-21adfe8962f2 -- S3
+        b083f8ae-42fb-41f0-a2a3-4e7c9f93aadb -- [guide]
+        bf021ce9-465b-4419-86e7-bddfd208fca4 -- git@newzaphod:~/repositories/annex/photos.git [zaphod]
+ok
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+I trust Git Annex to keep hundreds of GB of data safe, and it has never failed me - despite my best efforts

From 18c4674550f34654b62319acac1c35f33575dafa Mon Sep 17 00:00:00 2001
From: binx <binx@web>
Date: Wed, 23 Nov 2016 18:01:06 +0000
Subject: [PATCH 099/367]

---
 ...o-way_assistant_sync_with_ssh_special_remote.mdwn | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/doc/forum/two-way_assistant_sync_with_ssh_special_remote.mdwn b/doc/forum/two-way_assistant_sync_with_ssh_special_remote.mdwn
index 749a7937a2..ca04e442ce 100644
--- a/doc/forum/two-way_assistant_sync_with_ssh_special_remote.mdwn
+++ b/doc/forum/two-way_assistant_sync_with_ssh_special_remote.mdwn
@@ -1,20 +1,22 @@
 I am attempting to set up automatic two-way synchronization between my laptop and a server via ssh by running assistant on both machines. I want to have both machines be non-bare and unlocked.
 
-On the server:
+On the rhel server:
 
     $ mkdir ~/annex
-    $ cd annex
+    $ cd ~/annex
     $ git init
     $ git annex init u --version=6
     $ echo This is test file 1. >testfile1.txt
     $ git annex add testfile1.txt
     $ git annex sync
+    $ git remote add ml2 ssh://laptop/Users/username/annex
     $ git annex adjust --unlock
     $ git annex wanted . standard
     $ git annex group . client
 
-On my laptop:
+On my mac laptop:
 
+    $ cd ~/
     $ git clone ssh://server/home/username/annex
     $ cd annex
     $ git annex init ml2 --version=6
@@ -23,8 +25,8 @@ On my laptop:
     $ git annex wanted . standard
     $ git annex group . client
 
-Everything seems to work when I manually sync. When I run
+Everything seems to work when I manually sync. But when I run
 
     $ git annex assistant
 
-on both machines, however, I only get one-way automatic synchronization. Changes on the laptop are immediately propagated to the server. But changes on the server do not show up on the laptop until I manually sync. What am I doing wrong?
+on both machines, I only get one-way automatic synchronization. Changes on the laptop are immediately propagated to the server. But changes on the server do not show up on the laptop until I manually sync. What am I doing wrong?

From ff4b93906db59fa99ead60308ef453d1ca02b24d Mon Sep 17 00:00:00 2001
From: "https://launchpad.net/~stephane-gourichon-lpad"
 <stephane-gourichon-lpad@web>
Date: Wed, 23 Nov 2016 18:04:27 +0000
Subject: [PATCH 100/367] Added a comment: Known bug, fixed.

---
 ...t_2_13c242250d1509d933b8f0bcb7b67302._comment | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 doc/bugs/Single_space_in_file_name_causes_git_annex_add_to_fail/comment_2_13c242250d1509d933b8f0bcb7b67302._comment

diff --git a/doc/bugs/Single_space_in_file_name_causes_git_annex_add_to_fail/comment_2_13c242250d1509d933b8f0bcb7b67302._comment b/doc/bugs/Single_space_in_file_name_causes_git_annex_add_to_fail/comment_2_13c242250d1509d933b8f0bcb7b67302._comment
new file mode 100644
index 0000000000..bf41d40a5a
--- /dev/null
+++ b/doc/bugs/Single_space_in_file_name_causes_git_annex_add_to_fail/comment_2_13c242250d1509d933b8f0bcb7b67302._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~stephane-gourichon-lpad"
+ nickname="stephane-gourichon-lpad"
+ avatar="http://cdn.libravatar.org/avatar/02d4a0af59175f9123720b4481d55a769ba954e20f6dd9b2792217d9fa0c6089"
+ subject="Known bug, fixed."
+ date="2016-11-23T18:04:27Z"
+ content="""
+This is a known bug introduced in 6.20161012 and fixed in 6.20161031.
+
+Solution is: just update your copy of git-annex. At this time most recent is 6.20161119 .
+
+For more details, see changelog at https://github.com/joeyh/git-annex/blob/master/CHANGELOG#L53
+
+
+
+"""]]

From 5c848b722534b6796fcdfcccdac70817c1a55f35 Mon Sep 17 00:00:00 2001
From: "scottgorlin@a32946b2aad278883c1690a0753241583a9855b9" <scottgorlin@web>
Date: Wed, 23 Nov 2016 20:07:46 +0000
Subject: [PATCH 101/367] Added a comment: IgnoreUnknown Include considered
 harmful?

---
 ..._2_32e142afd9fe65843d53883ba2ae48cb._comment | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 doc/bugs/git_annex_init_failed_due_to_unsupported_ssh_option/comment_2_32e142afd9fe65843d53883ba2ae48cb._comment

diff --git a/doc/bugs/git_annex_init_failed_due_to_unsupported_ssh_option/comment_2_32e142afd9fe65843d53883ba2ae48cb._comment b/doc/bugs/git_annex_init_failed_due_to_unsupported_ssh_option/comment_2_32e142afd9fe65843d53883ba2ae48cb._comment
new file mode 100644
index 0000000000..42550d51f4
--- /dev/null
+++ b/doc/bugs/git_annex_init_failed_due_to_unsupported_ssh_option/comment_2_32e142afd9fe65843d53883ba2ae48cb._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="scottgorlin@a32946b2aad278883c1690a0753241583a9855b9"
+ nickname="scottgorlin"
+ avatar="http://cdn.libravatar.org/avatar/2dd1fc8add62bbf4ffefac081b322563"
+ subject="IgnoreUnknown Include considered harmful?"
+ date="2016-11-23T20:07:45Z"
+ content="""
+As noted, include appears to not work on a mac at the moment.  This means git-annex silently ignores the included configs, which may be required to ssh to the remotes of interest.  This is happening to me.
+
+My understanding is that ssh aliases are the recommended way of juggling multiple private keys amongst multiple hosts, so it is a required part of many git workflows.  In this particular case, I have set up git annex on a NAS which does not allow multiple ssh users (QNAP) and the authentication is done only via key identity, not username.  Thus, host aliases are necessary.
+
+If one config can't include another, I would prefer an early failure indicating a problem with the config file, or better, a solution where git-annex doesn't require a config.  In this scenario, git fetch remote_name and git annex copy --to remotename do not resolve to the same alias definitions (the latter is missing because of the ignored config!).
+
+I got my setup to work only by finding and manually editing <repo>/.git/annex/ssh_config, which to my knowledge is undocumented (ie when is it written? do any commands change it?); manual mucking around inside .git to me is not a good practice, and for now I have two different alias's defined (in repo and in ~/.ssh/config)
+
+
+"""]]

From 27585f75698af226f287ce1f607083380106f214 Mon Sep 17 00:00:00 2001
From: StephaneGourichon <StephaneGourichon@web>
Date: Thu, 24 Nov 2016 11:27:59 +0000
Subject: [PATCH 102/367] Added a comment: Walkthrough of a prudent retroactive
 annex.

---
 ..._834410421ccede5194bd8fbaccea8d1a._comment | 82 +++++++++++++++++++
 1 file changed, 82 insertions(+)
 create mode 100644 doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_8_834410421ccede5194bd8fbaccea8d1a._comment

diff --git a/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_8_834410421ccede5194bd8fbaccea8d1a._comment b/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_8_834410421ccede5194bd8fbaccea8d1a._comment
new file mode 100644
index 0000000000..2c36962aa5
--- /dev/null
+++ b/doc/tips/How_to_retroactively_annex_a_file_already_in_a_git_repo/comment_8_834410421ccede5194bd8fbaccea8d1a._comment
@@ -0,0 +1,82 @@
+[[!comment format=mdwn
+ username="StephaneGourichon"
+ avatar="http://cdn.libravatar.org/avatar/8cea01af2c7a8bf529d0a3d918ed4abf"
+ subject="Walkthrough of a prudent retroactive annex."
+ date="2016-11-24T11:27:59Z"
+ content="""
+Been using the one-liner.  Despite the warning, I'm not dead yet.
+
+There's much more to do than the one-liner.
+
+This post offers instructions.
+
+# First simple try: slow
+
+Was slow (estimated >600s for 189 commits).
+
+# In tmpfs: about 6 times faster
+
+I have cloned repository into /run/user/1000/rewrite-git, which is a tmpfs mount point. (Machine has plenty of RAM.)
+
+There I also did `git annex init`, git-annex found its state branches.
+
+On second try I also did 
+
+    git checkout -t remotes/origin/synced/master
+
+So that filter-branch would clean that, too.
+
+There, `filter-branch` operation finished in 90s first try, 149s second try.
+
+`.git/objects` wasn't smaller.
+
+# Practicing reduction on clone
+
+This produced no visible benefit:
+
+time git gc --aggressive
+time git repack -a -d 
+
+Even cloning and retrying on clone. Oh, but I should have done `git clone file:///path` as said on git-filter-branch man page's section titled \"CHECKLIST FOR SHRINKING A REPOSITORY\" 
+
+This (as seen on https://rtyley.github.io/bfg-repo-cleaner/ ) was efficient:
+
+    git reflog expire --expire=now --all && git gc --prune=now --aggressive
+
+`.git/objects` shrunk from 148M to 58M
+
+All this was on a clone of the repo in tmpfs.
+
+# Propagating cleaned up branches to origin
+
+This confirmed that filter-branch did not change last tree:
+
+    git diff remotes/origin/master..master
+    git diff remotes/origin/synced/master synced/master
+
+This, expectedly, was refused:
+
+    git push origin master
+    git push origin synced/master
+
+On origin, I checked out the hash of current master, then on tmpfs clone
+
+    git push -f origin master
+    git push -f origin synced/master
+
+Looks good.
+
+I'm not doing the aggressive shrink now, because of the \"two orders of magnitude more caution than normal filter-branch\" recommended by arand.
+
+# Now what? Check if precious not broken
+
+I'm planning to do the same operation on the other repos, then :
+
+* if everything seems right, 
+* if `git annex sync` works between all those fellows
+* etc, 
+* then I would perform the reflog expire, gc prune on some then all of them, etc.
+
+Joey, does this seem okay?  Any comment?
+
+"""]]

From 9e2073f331ab057d6ac6eeb48bb2998e80ad3b96 Mon Sep 17 00:00:00 2001
From: Markus Hauru <markus@mhauru.org>
Date: Wed, 23 Nov 2016 22:08:04 -0500
Subject: [PATCH 103/367] Fixed typo in Schedule.hs.

---
 Command/Schedule.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Schedule.hs b/Command/Schedule.hs
index 5cc8b37bf5..c9d4f915f7 100644
--- a/Command/Schedule.hs
+++ b/Command/Schedule.hs
@@ -29,7 +29,7 @@ start = parse
   where
 	parse (name:[]) = go name performGet
 	parse (name:expr:[]) = go name $ \uuid -> do
-		showStart "schedile" name
+		showStart "schedule" name
 		performSet expr uuid
 	parse _ = giveup "Specify a repository."
 

From d1ad4f1c00a58da5d385be2aea0e0b80064013d0 Mon Sep 17 00:00:00 2001
From: "davidriod@e75b369a4b1cced29c14354bce7493c61f00b1c7" <davidriod@web>
Date: Thu, 24 Nov 2016 19:23:43 +0000
Subject: [PATCH 104/367] Added a comment: Sharing rsync special remote between
 repository

---
 .../comment_14_2261b1b7441eff9e28ec8e1f98d77980._comment | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/special_remotes/rsync/comment_14_2261b1b7441eff9e28ec8e1f98d77980._comment

diff --git a/doc/special_remotes/rsync/comment_14_2261b1b7441eff9e28ec8e1f98d77980._comment b/doc/special_remotes/rsync/comment_14_2261b1b7441eff9e28ec8e1f98d77980._comment
new file mode 100644
index 0000000000..4d6f0cfc28
--- /dev/null
+++ b/doc/special_remotes/rsync/comment_14_2261b1b7441eff9e28ec8e1f98d77980._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="davidriod@e75b369a4b1cced29c14354bce7493c61f00b1c7"
+ nickname="davidriod"
+ avatar="http://cdn.libravatar.org/avatar/d6e327bd88b88802d6f0c20c83f682a2"
+ subject="Sharing rsync special remote between repository"
+ date="2016-11-24T19:23:42Z"
+ content="""
+I was wondering if it is possible to share a rsync special remote between repository which are not parented in any way. The use case would be that even if these repositories are not related at all they still may contains the same binary file. It would be useful to have a single rsync remote in order to reduce space usage. I think it could work as the object names are based on their checksum, but I wonder if anyone has already try that ?
+"""]]

From 3ed8895a096c8e79b3e7a1b3b43305c7c7d8b11d Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 24 Nov 2016 16:36:16 -0400
Subject: [PATCH 105/367] fix build

---
 CmdLine/GitRemoteTorAnnex.hs  | 4 ++--
 P2P/IO.hs                     | 2 +-
 RemoteDaemon/Transport/Tor.hs | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/CmdLine/GitRemoteTorAnnex.hs b/CmdLine/GitRemoteTorAnnex.hs
index 3282cc081a..3b2dcc0509 100644
--- a/CmdLine/GitRemoteTorAnnex.hs
+++ b/CmdLine/GitRemoteTorAnnex.hs
@@ -10,8 +10,8 @@ module CmdLine.GitRemoteTorAnnex where
 import Common
 import qualified Annex
 import qualified Git.CurrentRepo
-import Remote.Helper.P2P
-import Remote.Helper.P2P.IO
+import P2P.Protocol
+import P2P.IO
 import Remote.Helper.Tor
 import Utility.Tor
 import Utility.AuthToken
diff --git a/P2P/IO.hs b/P2P/IO.hs
index 9a1243f52b..822eb524ef 100644
--- a/P2P/IO.hs
+++ b/P2P/IO.hs
@@ -12,7 +12,7 @@ module P2P.IO
 	, runNetProtoHandle
 	) where
 
-import Remote.Helper.P2P
+import P2P.Protocol
 import Utility.Process
 import Git
 import Git.Command
diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs
index f8dfede104..6c5471966e 100644
--- a/RemoteDaemon/Transport/Tor.hs
+++ b/RemoteDaemon/Transport/Tor.hs
@@ -13,8 +13,8 @@ import RemoteDaemon.Common
 import Utility.Tor
 import Utility.FileMode
 import Remote.Helper.Tor
-import Remote.Helper.P2P
-import Remote.Helper.P2P.IO
+import P2P.Protocol
+import P2P.IO
 import Annex.UUID
 import Types.UUID
 import Messages

From 1b4bce0b0c7baa90ed7dae4072cd746a844c193c Mon Sep 17 00:00:00 2001
From: "edaneault@c9ac084da568ceb4d4566371a2da53f847258e7e" <edaneault@web>
Date: Fri, 25 Nov 2016 18:10:54 +0000
Subject: [PATCH 106/367]

---
 ...it-annex_link_to_different_file_names.mdwn | 41 +++++++++++++++++++
 1 file changed, 41 insertions(+)
 create mode 100644 doc/forum/Git-annex_link_to_different_file_names.mdwn

diff --git a/doc/forum/Git-annex_link_to_different_file_names.mdwn b/doc/forum/Git-annex_link_to_different_file_names.mdwn
new file mode 100644
index 0000000000..f6ce011738
--- /dev/null
+++ b/doc/forum/Git-annex_link_to_different_file_names.mdwn
@@ -0,0 +1,41 @@
+This is a recreation of a stackexchange question, in case the community here is more knowledgeable.
+
+Link to stackexchange question : http://unix.stackexchange.com/questions/325753/git-annex-link-to-different-file-names
+
+Content :
+"Maybe this is just a crazy use case that doesn't work, but I was wondering if there's a way to build a file's history from files with different file names. I'm exploring this idea because I'd like to have a git-annex system but I can't force my coworkers to adapt.
+
+Here's what I have in mind :
+
+    Folder 1, managed by coworkers (On a shared disk) :
+
+        drawing_shop_12_nov_2015.pdf
+        drawing_shop_13_nov_2015.pdf
+        drawing_asbuilt_14_nov_2015.pdf
+        drawing_asbuilt_rev1_15_nov_2015.pdf
+
+And
+
+    Git-annex, managed by me :
+
+        drawing.pdf
+
+    (with a shop branch and a asbuilt branch)
+
+The git-annex's drawing.pdf would have an history like this :
+
+[shop]
+|
+Commit A "Initial shop drawing"
+|
+Commit B "Add corrections from Wizzbasket"
+ \
+  |
+  [asbuilt]
+  Commit C "Reflect as built"
+  |
+  Commit D "Change dweezelbox block for simplicity"
+
+But somehow the "managed by coworkers" repo would be a direct mode repo with Commit A pointing to drawing_shop_12_nov_2015.pdf, Commit B to drawing_shop_13_nov_2015.pdf etc.
+
+Can this be done?"

From afbc832eb8b057276bdb4debf53481f2879ee78f Mon Sep 17 00:00:00 2001
From: "edaneault@c9ac084da568ceb4d4566371a2da53f847258e7e" <edaneault@web>
Date: Fri, 25 Nov 2016 18:12:39 +0000
Subject: [PATCH 107/367]

---
 ...it-annex_link_to_different_file_names.mdwn | 22 +++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/doc/forum/Git-annex_link_to_different_file_names.mdwn b/doc/forum/Git-annex_link_to_different_file_names.mdwn
index f6ce011738..a7a7c2727c 100644
--- a/doc/forum/Git-annex_link_to_different_file_names.mdwn
+++ b/doc/forum/Git-annex_link_to_different_file_names.mdwn
@@ -24,17 +24,17 @@ And
 
 The git-annex's drawing.pdf would have an history like this :
 
-[shop]
-|
-Commit A "Initial shop drawing"
-|
-Commit B "Add corrections from Wizzbasket"
- \
-  |
-  [asbuilt]
-  Commit C "Reflect as built"
-  |
-  Commit D "Change dweezelbox block for simplicity"
+    [shop]
+    |
+    Commit A "Initial shop drawing"
+    |
+    Commit B "Add corrections from Wizzbasket"
+     \
+      |
+      [asbuilt]
+      Commit C "Reflect as built"
+      |
+      Commit D "Change dweezelbox block for simplicity"
 
 But somehow the "managed by coworkers" repo would be a direct mode repo with Commit A pointing to drawing_shop_12_nov_2015.pdf, Commit B to drawing_shop_13_nov_2015.pdf etc.
 

From 52ce3939b794c4df89edf1d0e5cf8a1551ecd880 Mon Sep 17 00:00:00 2001
From: CandyAngel <CandyAngel@web>
Date: Fri, 25 Nov 2016 20:27:07 +0000
Subject: [PATCH 108/367] Added a comment

---
 ...comment_4_2a9eb14a8c6d06747bb5dda7ff179ec7._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/addurl_pathdepth_description_misleading/comment_4_2a9eb14a8c6d06747bb5dda7ff179ec7._comment

diff --git a/doc/bugs/addurl_pathdepth_description_misleading/comment_4_2a9eb14a8c6d06747bb5dda7ff179ec7._comment b/doc/bugs/addurl_pathdepth_description_misleading/comment_4_2a9eb14a8c6d06747bb5dda7ff179ec7._comment
new file mode 100644
index 0000000000..0cafc2a4dc
--- /dev/null
+++ b/doc/bugs/addurl_pathdepth_description_misleading/comment_4_2a9eb14a8c6d06747bb5dda7ff179ec7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="CandyAngel"
+ avatar="http://cdn.libravatar.org/avatar/15c0aade8bec5bf004f939dd73cf9ed8"
+ subject="comment 4"
+ date="2016-11-25T20:27:07Z"
+ content="""
+I really don't know what to say. I can't even figure out which computer I updated git-annex on to test if it was still happening.. let alone reproduce it anymore. It does work fine.
+
+I'm so sorry to bother you with this, I've done something stupid! This is exactly why you ask for a transcript of bugs occurring. (Feel free to use this as an example for why you ask for them, so some good can come of it at least..).
+"""]]

From 07504166eda976ddcc42a4ca3f3c280b5fafb84c Mon Sep 17 00:00:00 2001
From: boh <boh@web>
Date: Sun, 27 Nov 2016 12:23:20 +0000
Subject: [PATCH 109/367] Added a comment

---
 ..._c46cdba62da4f5ccfdc42dfc33aec600._comment | 34 +++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_9_c46cdba62da4f5ccfdc42dfc33aec600._comment

diff --git a/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_9_c46cdba62da4f5ccfdc42dfc33aec600._comment b/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_9_c46cdba62da4f5ccfdc42dfc33aec600._comment
new file mode 100644
index 0000000000..a6a2397e7f
--- /dev/null
+++ b/doc/bugs/Assistant_drops_files_from_remote_repos_that_it_shouldn__39__t/comment_9_c46cdba62da4f5ccfdc42dfc33aec600._comment
@@ -0,0 +1,34 @@
+[[!comment format=mdwn
+ username="boh"
+ avatar="http://cdn.libravatar.org/avatar/e7fa2d1c5d95e323fe48887f7f827b1f"
+ subject="comment 9"
+ date="2016-11-27T12:23:20Z"
+ content="""
+Seems as if the problem still exists in 6.20161118 (Debian).
+
+I have three repositories (among others), `jolla`, `sts-3xx`, and `here`.  `jolla` and `here` are in group `manual`, `sts-3xx` is `backup`; `here` and `sts-3xx` have assistants running, `jolla` not.  `jolla` and `sts-3xx` have slightly older versions of git-annex installed.
+
+Now, when I copy a file from `here` to `jolla` like this
+
+    git annex copy real_programmers.png -t jolla
+
+the file is subsequently dropped by the assistant:
+
+```
+drop real_programmers.png (locking jolla...) [2016-11-27 13:00:02.667376556] chat: ssh [\"-S\",\".git/annex/ssh/jolla\",\"-o\",\"ControlMaster
+=auto\",\"-o\",\"ControlPersist=yes\",\"-F\",\".git/annex/ssh.config\",\"-T\",\"jolla\",\"git-annex-shell 'lockcontent' '/~/Music/media/' '--debug' '
+SHA256E-s84499--ff98a733cc0122858fb11433c720e2d038fec190a3d36380d0e7e8dab468f883.png' --uuid 5298e3ce-1106-4d5e-b052-0aee4b27a344\"]
+(locking sts-3xx...) [2016-11-27 13:00:03.252473676] chat: ssh [..., \"git-annex-shell 'lockcontent' '/backups/exot/media/' '--debug' 'SHA256E-s84499--ff98a733cc0122858fb11433c720e2d038fec190a3d 36380d0e7e8dab468f883.png' --uuid 1fec6253-171d-4f86-885b-e233be2d65ec\"]
+(lockcontent failed) [2016-11-27 13:00:03.486158016] process done ExitFailure 1
+(checking sts-3xx...) [2016-11-27 13:00:03.487047149] call: ssh [..., \"git-annex-shell 'inannex' '/backups/exot/media/' '--debug' 'SHA256E-s84499--ff98a733cc0122858fb11433c720e2d038fec190a3d363 80d0e7e8dab468f883.png' --uuid 1fec6253-171d-4f86-885b-e233be2d65ec\"]
+[2016-11-27 13:00:03.76435136] process done ExitSuccess
+[2016-11-27 13:00:03.764705754] Dropping from here proof: Just (SafeDropProof (NumCopies 2) [RecentlyVerifiedCopy UUID \"1fec6253-171d-4 f86-885b-e233be2d65ec\",LockedCopy UUID \"5298e3ce-1106-4d5e-b052-0aee4b27a344\"] (Just (ContentRemovalLock (Key {keyName = \"ff98a733cc012 2858fb11433c720e2d038fec190a3d36380d0e7e8dab468f883.png\", keyBackendName = \"SHA256E\", keySize = Just 84499, keyMtime = Nothing, keyChun kSize = Nothing, keyChunkNum = Nothing}))))
+[2016-11-27 13:00:04.24333081] process done ExitFailure 1
+ok
+[2016-11-27 13:00:04.251232455] dropped real_programmers.png (from here) (copies now 4) : drop wanted after Upload UUID \"5298e3ce-1106- 4d5e-b052-0aee4b27a344\" real_programmers.png Just 84499
+```
+
+However, I failed to reproduce the problem by replicating my setup with fresh repositories …
+
+Please let me know if you need more information, and *so* many thanks for git-annex!
+"""]]

From 36409ab5cf80172cb8ad2acacedff770037a13ab Mon Sep 17 00:00:00 2001
From: annexuser <annexuser@web>
Date: Sun, 27 Nov 2016 20:39:26 +0000
Subject: [PATCH 110/367]

---
 ...serving_Directories_in_Metadata_Views.mdwn | 47 +++++++++++++++++++
 1 file changed, 47 insertions(+)
 create mode 100644 doc/forum/Preserving_Directories_in_Metadata_Views.mdwn

diff --git a/doc/forum/Preserving_Directories_in_Metadata_Views.mdwn b/doc/forum/Preserving_Directories_in_Metadata_Views.mdwn
new file mode 100644
index 0000000000..dfc45cb4b6
--- /dev/null
+++ b/doc/forum/Preserving_Directories_in_Metadata_Views.mdwn
@@ -0,0 +1,47 @@
+I want to use metadata views to sort files into top-level directories based on a tag, but then preserve the directory structure underneath that. I'm having trouble with this.
+
+Say I have an annex at `~/annex` with a structure like this:
+
+    $ tree
+    .
+    ├── foo
+    │   └── bar
+    │       ├── one.txt
+    │       ├── three.txt
+    │       └── two.txt
+    └── waldo
+        └── fred
+            ├── a.txt
+            ├── b.txt
+            └── c.txt
+
+I tag some of the files with `blah`:
+
+    $ git annex metadata -t blah foo/bar/*
+
+Now I want to change my view to only see those files with a certain tag, but I want to maintain their directory structure, ie I want to end up with something like this:
+
+    $ tree
+    .
+    ├── blah
+    │   └── foo
+    │       └── bar
+    │           ├── one.txt
+    │           ├── three.txt
+    │           └── two.txt
+
+If I do `git annex view blah` I see the files `one.txt`, `two.txt` and `three.txt` but they are in the top level of `~/annex`. The `foo` and `bar` directories are not present.
+
+If I do `git annex view blah "/=*"` then the files I present under the `foo` directory, but the `bar` subdirectory is not there.
+
+It would also be fine if I could just hide the files that did not have the `blah` tag, so that I ended up with this:
+
+    $ tree
+    .
+    ├── foo
+    │   └── bar
+    │       ├── one.txt
+    │       ├── three.txt
+    │       └── two.txt
+
+Is something like this possible?

From f4d6c34f6510df263815fc3eff411c7a5bdd0c97 Mon Sep 17 00:00:00 2001
From: "https://launchpad.net/~felixonmars" <felixonmars@web>
Date: Mon, 28 Nov 2016 04:12:18 +0000
Subject: [PATCH 111/367] Added a comment

---
 ..._ada25b56fe50225e518a46cb38247483._comment | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 doc/bugs/cabal_constraints_for_aws_and_esqueleto/comment_1_ada25b56fe50225e518a46cb38247483._comment

diff --git a/doc/bugs/cabal_constraints_for_aws_and_esqueleto/comment_1_ada25b56fe50225e518a46cb38247483._comment b/doc/bugs/cabal_constraints_for_aws_and_esqueleto/comment_1_ada25b56fe50225e518a46cb38247483._comment
new file mode 100644
index 0000000000..ced3a97b0e
--- /dev/null
+++ b/doc/bugs/cabal_constraints_for_aws_and_esqueleto/comment_1_ada25b56fe50225e518a46cb38247483._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~felixonmars"
+ nickname="felixonmars"
+ avatar="http://cdn.libravatar.org/avatar/17284a3bb2e4ad9d3be8fab31f49865be9c1dc22143c728de731fe800a335d38"
+ subject="comment 1"
+ date="2016-11-28T04:12:18Z"
+ content="""
+aws has merged a PR to support http-conduit 2.2, but git-annex itself doesn't build with the new component yet:
+
+[ 95 of 544] Compiling Utility.Url      ( Utility/Url.hs, dist/build/git-annex/git-annex-tmp/Utility/Url.o )
+
+Utility/Url.hs:354:34: error:
+    * The constructor `StatusCodeException' should have 2 arguments, but has been given 3
+    * In the pattern: StatusCodeException s _ _
+      In an equation for `matchStatusCodeException':
+          matchStatusCodeException want e@(StatusCodeException s _ _)
+            | want s = Just e
+            | otherwise = Nothing
+
+Utility/Url.hs:354:34: error:
+    * Couldn't match expected type `HttpException'
+                  with actual type `HttpExceptionContent'
+    * In the pattern: StatusCodeException s _ _
+      In an equation for `matchStatusCodeException':
+          matchStatusCodeException want e@(StatusCodeException s _ _)
+            | want s = Just e
+            | otherwise = Nothing
+"""]]

From 02dd09a25db01842717a50a150cb9391e1e3f42f Mon Sep 17 00:00:00 2001
From: "https://launchpad.net/~felixonmars" <felixonmars@web>
Date: Mon, 28 Nov 2016 04:16:22 +0000
Subject: [PATCH 112/367] removed

---
 ..._ada25b56fe50225e518a46cb38247483._comment | 28 -------------------
 1 file changed, 28 deletions(-)
 delete mode 100644 doc/bugs/cabal_constraints_for_aws_and_esqueleto/comment_1_ada25b56fe50225e518a46cb38247483._comment

diff --git a/doc/bugs/cabal_constraints_for_aws_and_esqueleto/comment_1_ada25b56fe50225e518a46cb38247483._comment b/doc/bugs/cabal_constraints_for_aws_and_esqueleto/comment_1_ada25b56fe50225e518a46cb38247483._comment
deleted file mode 100644
index ced3a97b0e..0000000000
--- a/doc/bugs/cabal_constraints_for_aws_and_esqueleto/comment_1_ada25b56fe50225e518a46cb38247483._comment
+++ /dev/null
@@ -1,28 +0,0 @@
-[[!comment format=mdwn
- username="https://launchpad.net/~felixonmars"
- nickname="felixonmars"
- avatar="http://cdn.libravatar.org/avatar/17284a3bb2e4ad9d3be8fab31f49865be9c1dc22143c728de731fe800a335d38"
- subject="comment 1"
- date="2016-11-28T04:12:18Z"
- content="""
-aws has merged a PR to support http-conduit 2.2, but git-annex itself doesn't build with the new component yet:
-
-[ 95 of 544] Compiling Utility.Url      ( Utility/Url.hs, dist/build/git-annex/git-annex-tmp/Utility/Url.o )
-
-Utility/Url.hs:354:34: error:
-    * The constructor `StatusCodeException' should have 2 arguments, but has been given 3
-    * In the pattern: StatusCodeException s _ _
-      In an equation for `matchStatusCodeException':
-          matchStatusCodeException want e@(StatusCodeException s _ _)
-            | want s = Just e
-            | otherwise = Nothing
-
-Utility/Url.hs:354:34: error:
-    * Couldn't match expected type `HttpException'
-                  with actual type `HttpExceptionContent'
-    * In the pattern: StatusCodeException s _ _
-      In an equation for `matchStatusCodeException':
-          matchStatusCodeException want e@(StatusCodeException s _ _)
-            | want s = Just e
-            | otherwise = Nothing
-"""]]

From 7982826f895234f51670d3849a73449141dead80 Mon Sep 17 00:00:00 2001
From: "https://launchpad.net/~felixonmars" <felixonmars@web>
Date: Mon, 28 Nov 2016 04:17:13 +0000
Subject: [PATCH 113/367] Added a comment

---
 ..._d91e44573ef4a0ec6e7098cb4cd360f5._comment | 30 +++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 doc/bugs/cabal_constraints_for_aws_and_esqueleto/comment_1_d91e44573ef4a0ec6e7098cb4cd360f5._comment

diff --git a/doc/bugs/cabal_constraints_for_aws_and_esqueleto/comment_1_d91e44573ef4a0ec6e7098cb4cd360f5._comment b/doc/bugs/cabal_constraints_for_aws_and_esqueleto/comment_1_d91e44573ef4a0ec6e7098cb4cd360f5._comment
new file mode 100644
index 0000000000..327b634694
--- /dev/null
+++ b/doc/bugs/cabal_constraints_for_aws_and_esqueleto/comment_1_d91e44573ef4a0ec6e7098cb4cd360f5._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~felixonmars"
+ nickname="felixonmars"
+ avatar="http://cdn.libravatar.org/avatar/17284a3bb2e4ad9d3be8fab31f49865be9c1dc22143c728de731fe800a335d38"
+ subject="comment 1"
+ date="2016-11-28T04:17:12Z"
+ content="""
+aws has merged a PR to support http-conduit 2.2, but git-annex itself doesn't build with the new component yet:
+
+```
+[ 95 of 544] Compiling Utility.Url      ( Utility/Url.hs, dist/build/git-annex/git-annex-tmp/Utility/Url.o )
+
+Utility/Url.hs:354:34: error:
+    * The constructor `StatusCodeException' should have 2 arguments, but has been given 3
+    * In the pattern: StatusCodeException s _ _
+      In an equation for `matchStatusCodeException':
+          matchStatusCodeException want e@(StatusCodeException s _ _)
+            | want s = Just e
+            | otherwise = Nothing
+
+Utility/Url.hs:354:34: error:
+    * Couldn't match expected type `HttpException'
+                  with actual type `HttpExceptionContent'
+    * In the pattern: StatusCodeException s _ _
+      In an equation for `matchStatusCodeException':
+          matchStatusCodeException want e@(StatusCodeException s _ _)
+            | want s = Just e
+            | otherwise = Nothing
+```
+"""]]

From 5b9615d3610279d43cc7992bc178e9b99539132f Mon Sep 17 00:00:00 2001
From: "palday@91f366b5178879146d2b6e1e53bfa21389ee89a8" <palday@web>
Date: Tue, 29 Nov 2016 02:17:52 +0000
Subject: [PATCH 114/367] Added a comment: Temporary workaround until the brew
 formula is updated

---
 ...nt_1_45003ab569c4649ca29c07877a83af29._comment | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/bugs/regression_due_to_usage_of_ssh_7.3___34__include__34___feature/comment_1_45003ab569c4649ca29c07877a83af29._comment

diff --git a/doc/bugs/regression_due_to_usage_of_ssh_7.3___34__include__34___feature/comment_1_45003ab569c4649ca29c07877a83af29._comment b/doc/bugs/regression_due_to_usage_of_ssh_7.3___34__include__34___feature/comment_1_45003ab569c4649ca29c07877a83af29._comment
new file mode 100644
index 0000000000..566926a321
--- /dev/null
+++ b/doc/bugs/regression_due_to_usage_of_ssh_7.3___34__include__34___feature/comment_1_45003ab569c4649ca29c07877a83af29._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="palday@91f366b5178879146d2b6e1e53bfa21389ee89a8"
+ nickname="palday"
+ avatar="http://cdn.libravatar.org/avatar/077a63af75ddba159980fbf88690f401"
+ subject="Temporary workaround until the brew formula is updated"
+ date="2016-11-29T02:17:52Z"
+ content="""
+The homebrew formula doesn't yet this fix, but you can get around the problem in the meantime by getting a newer SSH via homebrew:
+
+```
+brew install homebrew/dupes/openssh
+```
+
+You can then choose to keep that or get rid of it when the formula for git annex is later updated.
+"""]]

From 91e33d6c957b926357ae1faac674213c67928ac7 Mon Sep 17 00:00:00 2001
From: db48x <db48x@web>
Date: Tue, 29 Nov 2016 12:24:43 +0000
Subject: [PATCH 115/367]

---
 ...it-annex_fromkey_barfs_on_utf-8_input.mdwn | 34 +++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 doc/bugs/git-annex_fromkey_barfs_on_utf-8_input.mdwn

diff --git a/doc/bugs/git-annex_fromkey_barfs_on_utf-8_input.mdwn b/doc/bugs/git-annex_fromkey_barfs_on_utf-8_input.mdwn
new file mode 100644
index 0000000000..c1f71789b9
--- /dev/null
+++ b/doc/bugs/git-annex_fromkey_barfs_on_utf-8_input.mdwn
@@ -0,0 +1,34 @@
+### Please describe the problem.
+
+I'm sending a stream of keys and filenames to git-annex fromkey on stdin, and it errors out with "git-annex: <stdin>: hGetContents: invalid argument (invalid byte sequence)". On the other hand yipdw tried to reproduce this and it worked fine for him, so I must be doing something wrong.
+
+I have LANG=en_US.UTF-8 set in my environment, if that matters.
+
+### What steps will reproduce the problem?
+
+[[!format sh """
+echo "MD5-s3263532--0b4d070eff7baa8ef314ca330aecb71f é" | git-annex fromkey
+"""]]
+
+### What version of git-annex are you using? On what operating system?
+
+[[!format sh """
+git-annex version: 6.20161118-g0a34f08
+build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+local repository version: 5
+supported repository versions: 3 5 6
+upgrade supported from repository versions: 0 1 2 3 4 5
+operating system: linux x86_64
+"""]]
+
+### Please provide any additional information below.
+
+Note that this is indeed valid utf-8:
+
+[[!format sh """
+ db48x  ~  projects  IA.BAK-server  echo "é" | hexdump -C
+00000000  c3 a9 0a                                          |...|
+00000003
+"""]]

From c68d4305eb016e026d655dff7c7cb96cbb8296ea Mon Sep 17 00:00:00 2001
From: "senatorzergling@c1d78877a6120a80f0f4005210eda262c1586555"
 <senatorzergling@web>
Date: Tue, 29 Nov 2016 15:02:47 +0000
Subject: [PATCH 116/367] Added link to Windows installation instructions. The
 page was not easy to find.

---
 doc/install.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index dccd9e52ea..210493158c 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -17,7 +17,7 @@ detailed instructions             | quick install
 &nbsp;&nbsp;[[ScientificLinux5]]  |
 &nbsp;&nbsp;[[openSUSE]]          | `zypper in git-annex`
 &nbsp;&nbsp;[[Docker]]            | 
-[[Windows]]                       | [download installer](https://downloads.kitenet.net/git-annex/windows/current/) **beta**
+[[Windows]]                       | [download installer](https://downloads.kitenet.net/git-annex/windows/current/) and follow the [instructions here](http://git-annex.branchable.com/install/Windows/) **beta**
 """]]
 
 All the download links above use https for security. For added security, see

From fe310d1c35c31103f53aa1f79aae3b2d4d944822 Mon Sep 17 00:00:00 2001
From: "senatorzergling@c1d78877a6120a80f0f4005210eda262c1586555"
 <senatorzergling@web>
Date: Tue, 29 Nov 2016 15:13:16 +0000
Subject: [PATCH 117/367] Added more explanations of my experience with the
 Windows install process.

---
 doc/install/Windows.mdwn | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/doc/install/Windows.mdwn b/doc/install/Windows.mdwn
index 7ea667a10e..00ef702d4d 100644
--- a/doc/install/Windows.mdwn
+++ b/doc/install/Windows.mdwn
@@ -2,8 +2,12 @@ git-annex now does Windows!
 
 * First, [install Git for Windows](http://git-scm.com/downloads)  
   Important: **Get the 32 bit version not the 64 bit version.**  
-  (Note that msysgit is no longer supported.)
+  (Note that msysgit is no longer supported.) If you installed the
+  64 bit version of git, then parts of git-annex will still run,
+  however, some features, including tools like rsync, are not-functional.
 * Then, [install git-annex](https://downloads.kitenet.net/git-annex/windows/current/)
+  into the same location Git has been installed to. You must provide this path to the
+  git-annex installer. For instance, the path may be "C:\Users\John\AppData\Local\Programs\Git\"
 
 This port is now in reasonably good shape for command-line use of
 git-annex. The assistant and webapp are also usable. There are some known

From c8d2595758ae803f1ce617056158632e28c5be28 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 29 Nov 2016 12:05:37 -0400
Subject: [PATCH 118/367] Windows autobuilder is going to be hosted at
 Dartmouth now

---
 doc/install/OSX.mdwn     | 2 +-
 doc/install/Windows.mdwn | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index 10fc8bad43..082fd517d6 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -18,7 +18,7 @@ several more. Handy if you don't otherwise have git installed.
 
 ## autobuilds
 
-Thanks to Dartmouth for hosting the autobuilder.
+Thanks to Dartmouth College for hosting the autobuilder.
 
 * [autobuild of git-annex.dmg](https://downloads.kitenet.net/git-annex/autobuild/x86_64-apple-yosemite/git-annex.dmg) ([build logs](https://downloads.kitenet.net/git-annex/autobuild/x86_64-apple-yosemite/))
 
diff --git a/doc/install/Windows.mdwn b/doc/install/Windows.mdwn
index 7ea667a10e..f079bf8cd8 100644
--- a/doc/install/Windows.mdwn
+++ b/doc/install/Windows.mdwn
@@ -18,7 +18,7 @@ important thing is that it should end with "All tests passed".
 ## autobuilds
 
 A daily build is also available, thanks to Yury V. Zaytsev and
-[NEST](http://nest-initiative.org/).
+Dartmouth College.
 
 * [download](https://downloads.kitenet.net/git-annex/autobuild/windows/) ([build logs](https://qa.nest-initiative.org/view/msysGit/job/msysgit-git-annex-assistant-test/))
 

From 87bd25fd51403a0a288c471638a17c6b177e7903 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 29 Nov 2016 13:32:20 -0400
Subject: [PATCH 119/367] Revert "Added link to Windows installation
 instructions. The page was not easy to find."

This reverts commit c68d4305eb016e026d655dff7c7cb96cbb8296ea.

With all due respect, the link to the instructions for each OS is in the
"detailed instructions" column. Adding a second link to the same thing
is unlikely to help.
---
 doc/install.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index 210493158c..dccd9e52ea 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -17,7 +17,7 @@ detailed instructions             | quick install
 &nbsp;&nbsp;[[ScientificLinux5]]  |
 &nbsp;&nbsp;[[openSUSE]]          | `zypper in git-annex`
 &nbsp;&nbsp;[[Docker]]            | 
-[[Windows]]                       | [download installer](https://downloads.kitenet.net/git-annex/windows/current/) and follow the [instructions here](http://git-annex.branchable.com/install/Windows/) **beta**
+[[Windows]]                       | [download installer](https://downloads.kitenet.net/git-annex/windows/current/) **beta**
 """]]
 
 All the download links above use https for security. For added security, see

From 84358624c6661504ae40719391c1bad23ba764c6 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 29 Nov 2016 13:37:16 -0400
Subject: [PATCH 120/367] tighten added language

Removed part about neeing to install git-annex and git in the same
location. The installer checks if the location given exists, and if not,
tells the user that git is not installed there. And the default should
work if the user just picks it in both installers.
---
 doc/install/Windows.mdwn | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/doc/install/Windows.mdwn b/doc/install/Windows.mdwn
index ff6a6adf4a..c9b60d6428 100644
--- a/doc/install/Windows.mdwn
+++ b/doc/install/Windows.mdwn
@@ -1,13 +1,13 @@
 git-annex now does Windows!
 
-* First, [install Git for Windows](http://git-scm.com/downloads)  
+* First, [install Git for Windows](http://git-scm.com/downloads)
+
   Important: **Get the 32 bit version not the 64 bit version.**  
-  (Note that msysgit is no longer supported.) If you installed the
-  64 bit version of git, then parts of git-annex will still run,
-  however, some features, including tools like rsync, are not-functional.
+  If you installed the 64 bit version of git, then parts of git-annex will
+  still run, however, some features, including tools like rsync, will
+  not work.
+
 * Then, [install git-annex](https://downloads.kitenet.net/git-annex/windows/current/)
-  into the same location Git has been installed to. You must provide this path to the
-  git-annex installer. For instance, the path may be "C:\Users\John\AppData\Local\Programs\Git\"
 
 This port is now in reasonably good shape for command-line use of
 git-annex. The assistant and webapp are also usable. There are some known

From 53b6d9057c1215b124f0af68b0b710d70efce66c Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 29 Nov 2016 13:02:19 -0400
Subject: [PATCH 121/367] move tor hidden service socket to /etc, temporarily
 violating the FHS

On Debian, apparmor prevents tor from reading from most locations. And,
it silently fails if it is prevented from reading the hidden service
socket. I filed #846275 about this; violating the FHS is the least bad of a
bad set of choices until that bug is fixed.
---
 Utility/Tor.hs | 62 ++++++++++++++++++++++++++++++++++----------------
 1 file changed, 43 insertions(+), 19 deletions(-)

diff --git a/Utility/Tor.hs b/Utility/Tor.hs
index 0900fb87e7..3b9ddb6a62 100644
--- a/Utility/Tor.hs
+++ b/Utility/Tor.hs
@@ -9,6 +9,8 @@ module Utility.Tor where
 
 import Common
 import Utility.ThreadScheduler
+import Utility.FileMode
+
 import System.PosixCompat.Types
 import Data.Char
 import Network.Socket
@@ -27,12 +29,7 @@ type UniqueIdent = String
 
 connectHiddenService :: OnionAddress -> OnionPort -> IO Socket
 connectHiddenService (OnionAddress address) port = do
-        soc <- socket AF_UNIX Stream defaultProtocol
-	connect soc (SockAddrUnix "/run/user/1000/1ecd1f64-3234-47ec-876c-47c4bd7f7407.sock")
-	return soc
-
-connectHiddenService' :: OnionAddress -> OnionPort -> IO Socket
-connectHiddenService' (OnionAddress address) port = do
+	hPutStrLn stderr $ show ("connect to", address, port)
 	(s, _) <- socksConnect torsockconf socksaddr
 	return s
   where
@@ -54,6 +51,7 @@ connectHiddenService' (OnionAddress address) port = do
 -- identifier, returns its information without making any changes.
 addHiddenService :: UserID -> UniqueIdent -> IO (OnionAddress, OnionPort)
 addHiddenService uid ident = do
+	prepHiddenServiceSocketDir uid ident
 	ls <- lines <$> readFile torrc
 	let portssocks = mapMaybe (parseportsock . separate isSpace) ls
 	case filter (\(_, s) -> s == sockfile) portssocks of
@@ -84,7 +82,7 @@ addHiddenService uid ident = do
 		return (p, drop 1 (dropWhile (/= ':') l))
 	parseportsock _ = Nothing
 	
-	sockfile = socketFile uid ident
+	sockfile = hiddenServiceSocketFile uid ident
 
 	-- An infinite random list of high ports.
 	highports g = 
@@ -96,26 +94,52 @@ addHiddenService uid ident = do
 	waithiddenservice n p = do
 		v <- tryIO $ readFile $ hiddenServiceHostnameFile uid ident
 		case v of
-			Right s | ".onion\n" `isSuffixOf` s -> 
+			Right s | ".onion\n" `isSuffixOf` s ->
 				return (OnionAddress (takeWhile (/= '\n') s), p)
 			_ -> do
 				threadDelaySeconds (Seconds 1)
 				waithiddenservice (n-1) p
 
+-- | A hidden service directory to use.
+--
+-- The "hs" is used in the name to prevent too long a path name,
+-- which could present problems for socketFile.
+hiddenServiceDir :: UserID -> UniqueIdent -> FilePath
+hiddenServiceDir uid ident = libDir </> "hs_" ++ show uid ++ "_" ++ ident
+
+hiddenServiceHostnameFile :: UserID -> UniqueIdent -> FilePath
+hiddenServiceHostnameFile uid ident = hiddenServiceDir uid ident </> "hostname"
+
+-- | Location of the socket for a hidden service.
+--
+-- This has to be a location that tor can read from, and that the user
+-- can write to. Tor is often prevented by apparmor from reading
+-- from many locations. Putting it in /etc is a FHS violation, but it's the
+-- simplest and most robust choice until http://bugs.debian.org/846275 is
+-- dealt with.
+--
+-- Note that some unix systems limit socket paths to 92 bytes long.
+-- That should not be a problem if the UniqueIdent is around the length of
+-- a UUID.
+hiddenServiceSocketFile :: UserID -> UniqueIdent -> FilePath
+hiddenServiceSocketFile uid ident = etcDir </> "hidden_services" </> show uid ++ "_" ++ show ident </> "s"
+
+-- | Sets up the directory for the socketFile, with appropriate
+-- permissions. Must run as root.
+prepHiddenServiceSocketDir :: UserID -> UniqueIdent -> IO ()
+prepHiddenServiceSocketDir uid ident = do
+	createDirectoryIfMissing True d
+	setOwnerAndGroup d uid (-1)
+	modifyFileMode d $
+		addModes [ownerReadMode, ownerExecuteMode, ownerWriteMode]
+  where
+	d = takeDirectory $ hiddenServiceSocketFile uid ident
+
 torrc :: FilePath
 torrc = "/etc/tor/torrc"
 
 libDir :: FilePath
 libDir = "/var/lib/tor"
 
-runDir :: UserID -> FilePath
-runDir uid = "/var/run/user" </> show uid
-
-socketFile :: UserID -> UniqueIdent -> FilePath
-socketFile uid ident = runDir uid </> ident ++ ".sock"
-
-hiddenServiceDir :: UserID -> UniqueIdent -> FilePath
-hiddenServiceDir uid ident = libDir </> "hidden_service_" ++ show uid ++ "_" ++ ident
-
-hiddenServiceHostnameFile :: UserID -> UniqueIdent -> FilePath
-hiddenServiceHostnameFile uid ident = hiddenServiceDir uid ident </> "hostname"
+etcDir :: FilePath
+etcDir = "/etc/tor"

From 87b325de71ebd0033b6e99dc5491e08e523c6c15 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 29 Nov 2016 15:41:28 -0400
Subject: [PATCH 122/367] enable remotedaemon when assistant is not enabled

Needed to serve tor hidden services.
---
 CmdLine/GitAnnex.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/CmdLine/GitAnnex.hs b/CmdLine/GitAnnex.hs
index 0049ecb3c3..0fa14c98bb 100644
--- a/CmdLine/GitAnnex.hs
+++ b/CmdLine/GitAnnex.hs
@@ -101,6 +101,7 @@ import qualified Command.DiffDriver
 import qualified Command.Smudge
 import qualified Command.Undo
 import qualified Command.Version
+import qualified Command.RemoteDaemon
 #ifdef WITH_ASSISTANT
 import qualified Command.Watch
 import qualified Command.Assistant
@@ -110,7 +111,6 @@ import qualified Command.WebApp
 #ifdef WITH_XMPP
 import qualified Command.XMPPGit
 #endif
-import qualified Command.RemoteDaemon
 #endif
 import qualified Command.Test
 #ifdef WITH_TESTSUITE
@@ -209,6 +209,7 @@ cmds testoptparser testrunner =
 	, Command.Smudge.cmd
 	, Command.Undo.cmd
 	, Command.Version.cmd
+	, Command.RemoteDaemon.cmd
 #ifdef WITH_ASSISTANT
 	, Command.Watch.cmd
 	, Command.Assistant.cmd
@@ -218,7 +219,6 @@ cmds testoptparser testrunner =
 #ifdef WITH_XMPP
 	, Command.XMPPGit.cmd
 #endif
-	, Command.RemoteDaemon.cmd
 #endif
 	, Command.Test.cmd testoptparser testrunner
 #ifdef WITH_TESTSUITE

From 2ad3420c19f39f6b8457bd679792cd03221f3b74 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 29 Nov 2016 15:43:34 -0400
Subject: [PATCH 123/367] fix

---
 RemoteDaemon/Transport/Tor.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs
index 6c5471966e..ccb84d1e90 100644
--- a/RemoteDaemon/Transport/Tor.hs
+++ b/RemoteDaemon/Transport/Tor.hs
@@ -37,7 +37,7 @@ server th@(TransportHandle (LocalRepo r) _) = do
 
 	uid <- getRealUserID
 	let ident = fromUUID u
-	let sock = socketFile uid ident
+	let sock = hiddenServiceSocketFile uid ident
 	nukeFile sock
 	soc <- socket AF_UNIX Stream defaultProtocol
 	bind soc (SockAddrUnix sock)

From 38425fdc391e714fa2a90b13578e05254c72e486 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 29 Nov 2016 17:30:27 -0400
Subject: [PATCH 124/367] finish git-annex enable-tor

Make it stash the address away for git-annex p2p to use later, rather
than outputting it. And, look up the UUID itself.
---
 Command/EnableTor.hs          | 27 ++++++------
 Creds.hs                      |  1 +
 P2P/Address.hs                | 79 +++++++++++++++++++++++++++++++++++
 Utility/Tor.hs                | 10 ++---
 doc/git-annex-enable-tor.mdwn | 14 +++----
 git-annex.cabal               |  1 +
 6 files changed, 106 insertions(+), 26 deletions(-)
 create mode 100644 P2P/Address.hs

diff --git a/Command/EnableTor.hs b/Command/EnableTor.hs
index c581fa1d4b..d24ecb2dc7 100644
--- a/Command/EnableTor.hs
+++ b/Command/EnableTor.hs
@@ -8,27 +8,28 @@
 module Command.EnableTor where
 
 import Command
+import P2P.Address
 import Utility.Tor
+import Annex.UUID
 
 -- This runs as root, so avoid making any commits or initializing
--- git-annex, as that would create root-owned files.
+-- git-annex, or doing other things that create root-owned files.
 cmd :: Command
 cmd = noCommit $ dontCheck repoExists $
-	command "enable-tor" SectionSetup ""
-		"userid uuid" (withParams seek)
+	command "enable-tor" SectionSetup "enable tor hidden service"
+		"uid" (withParams seek)
 
 seek :: CmdParams -> CommandSeek
 seek = withWords start
 
-start :: CmdParams -> CommandStart
-start (suserid:uuid:[]) = case readish suserid of
-	Nothing -> error "Bad userid"
+start :: [String] -> CommandStart
+start ps = case readish =<< headMaybe ps of
+	Nothing -> giveup "Bad params"
 	Just userid -> do
-		(OnionAddress onionaddr, onionport) <- liftIO $
-			addHiddenService userid uuid
-		liftIO $ putStrLn $ 
-			"tor-annex::" ++
-			onionaddr ++ ":" ++ 
-			show onionport ++ " "
+		uuid <- getUUID
+		when (uuid == NoUUID) $
+			giveup "This can only be run in a git-annex repository."
+		(onionaddr, onionport) <- liftIO $
+			addHiddenService userid (fromUUID uuid)
+		storeP2PAddress $ TorAnnex onionaddr onionport
 		stop
-start _ = error "Bad params"
diff --git a/Creds.hs b/Creds.hs
index de3cd2a063..b5181aa1e3 100644
--- a/Creds.hs
+++ b/Creds.hs
@@ -15,6 +15,7 @@ module Creds (
 	getEnvCredPair,
 	writeCacheCreds,
 	readCacheCreds,
+	cacheCredsFile,
 	removeCreds,
 	includeCredsInfo,
 ) where
diff --git a/P2P/Address.hs b/P2P/Address.hs
new file mode 100644
index 0000000000..315219683f
--- /dev/null
+++ b/P2P/Address.hs
@@ -0,0 +1,79 @@
+{- P2P protocol addresses
+ -
+ - Copyright 2016 Joey Hess <id@joeyh.name>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module P2P.Address where
+
+import qualified Annex
+import Annex.Common
+import Git
+import Creds
+import Utility.AuthToken
+import Utility.Tor
+
+import qualified Data.Text as T
+
+-- | A P2P address, without an AuthToken.
+--
+-- This is enough information to connect to the peer,
+-- but not enough to authenticate with it.
+data P2PAddress = TorAnnex OnionAddress OnionPort
+	deriving (Eq, Show)
+
+-- | A P2P address, with an AuthToken
+data P2PAddressAuth = P2PAddressAuth P2PAddress AuthToken
+	deriving (Eq, Show)
+
+class FormatP2PAddress a where
+	formatP2PAddress :: a -> String
+	unformatP2PAddress :: String -> Maybe a
+
+instance FormatP2PAddress P2PAddress where
+	formatP2PAddress (TorAnnex (OnionAddress onionaddr) onionport) =
+		"tor-annex::" ++ onionaddr ++ ":" ++ show onionport
+	unformatP2PAddress s
+		| "tor-annex::" `isPrefixOf` s = do
+			let s' = dropWhile (== ':') $ dropWhile (/= ':') s
+			let (onionaddr, ps) = separate (== ':') s'
+			onionport <- readish ps
+			return (TorAnnex (OnionAddress onionaddr) onionport)
+		| otherwise = Nothing
+
+instance FormatP2PAddress P2PAddressAuth where
+	formatP2PAddress (P2PAddressAuth addr authtoken) =
+		formatP2PAddress addr ++ ":" ++ T.unpack (fromAuthToken authtoken)
+	unformatP2PAddress s = do
+		let (ra, rs) = separate (== ':') (reverse s)
+		addr <- unformatP2PAddress (reverse rs)
+		authtoken <- toAuthToken (T.pack $ reverse ra)
+		return (P2PAddressAuth addr authtoken)
+
+loadP2PAddresses :: Annex [P2PAddress]
+loadP2PAddresses = mapMaybe unformatP2PAddress . maybe [] lines
+	<$> readCacheCreds p2pAddressCredsFile
+
+storeP2PAddress :: P2PAddress -> Annex ()
+storeP2PAddress addr = do
+	addrs <- loadP2PAddresses
+	unless (addr `elem` addrs) $ do
+		let s = unlines $ map formatP2PAddress (addr:addrs)
+		let tmpnam = p2pAddressCredsFile ++ ".new"
+		writeCacheCreds s tmpnam
+		tmpf <- cacheCredsFile tmpnam
+		destf <- cacheCredsFile p2pAddressCredsFile
+		-- This may be run by root, so make the creds file
+		-- and directory have the same owner and group as
+		-- the git repository directory has.
+		st <- liftIO . getFileStatus =<< Annex.fromRepo repoLocation
+		let fixowner f = setOwnerAndGroup f (fileOwner st) (fileGroup st)
+		liftIO $ do
+			fixowner tmpf
+			fixowner (takeDirectory tmpf)
+			fixowner (takeDirectory (takeDirectory tmpf))
+			renameFile tmpf destf
+
+p2pAddressCredsFile :: FilePath
+p2pAddressCredsFile = "p2paddrs"
diff --git a/Utility/Tor.hs b/Utility/Tor.hs
index 3b9ddb6a62..e63bd82d4c 100644
--- a/Utility/Tor.hs
+++ b/Utility/Tor.hs
@@ -21,7 +21,7 @@ import qualified System.Random as R
 type OnionPort = Int
 
 newtype OnionAddress = OnionAddress String
-	deriving (Show)
+	deriving (Show, Eq)
 
 type OnionSocket = FilePath
 
@@ -57,7 +57,7 @@ addHiddenService uid ident = do
 	case filter (\(_, s) -> s == sockfile) portssocks of
 		((p, _s):_) -> waithiddenservice 1 p
 		_ -> do
-			highports <- R.getStdRandom highports
+			highports <- R.getStdRandom mkhighports
 			let newport = Prelude.head $
 				filter (`notElem` map fst portssocks) highports
 			writeFile torrc $ unlines $
@@ -74,7 +74,7 @@ addHiddenService uid ident = do
 				, ("sefvice", [Param "tor", Param "reload"])
 				]
 			unless reloaded $
-				error "failed to reload tor, perhaps the tor service is not running"
+				giveup "failed to reload tor, perhaps the tor service is not running"
 			waithiddenservice 120 newport
   where
 	parseportsock ("HiddenServicePort", l) = do
@@ -85,12 +85,12 @@ addHiddenService uid ident = do
 	sockfile = hiddenServiceSocketFile uid ident
 
 	-- An infinite random list of high ports.
-	highports g = 
+	mkhighports g = 
 		let (g1, g2) = R.split g
 		in (R.randomRs (1025, 65534) g1, g2)
 
 	waithiddenservice :: Int -> OnionPort -> IO (OnionAddress, OnionPort)
-	waithiddenservice 0 _ = error "tor failed to create hidden service, perhaps the tor service is not running"
+	waithiddenservice 0 _ = giveup "tor failed to create hidden service, perhaps the tor service is not running"
 	waithiddenservice n p = do
 		v <- tryIO $ readFile $ hiddenServiceHostnameFile uid ident
 		case v of
diff --git a/doc/git-annex-enable-tor.mdwn b/doc/git-annex-enable-tor.mdwn
index 9fb55db5f2..1c17380276 100644
--- a/doc/git-annex-enable-tor.mdwn
+++ b/doc/git-annex-enable-tor.mdwn
@@ -4,20 +4,18 @@ git-annex enable-tor - enable tor hidden service
 
 # SYNOPSIS
 
-git annex enable-tor userid uuid
+sudo git annex enable-tor $(id -u)
 
 # DESCRIPTION
 
-This plumbing-level command enables a tor hidden service for git-annex,
-using the specified repository uuid and userid. 
+This command enables a tor hidden service for git-annex.
 
-This command has to be run by root, since it modifies `/etc/tor/torrc`.
+It has to be run by root, since it modifies `/etc/tor/torrc`.
+Pass it your user id number, as output by `id -u`
 
 After this command is run, `git annex remotedaemon` can be run to serve the
-tor hidden service.
-
-Use the `git-annex p2p --gen-address` command to give other users access
-to your repository via the tor hidden service.
+tor hidden service, and then `git-annex p2p --gen-address` can be run to
+give other users access to your repository via the tor hidden service.
 
 # SEE ALSO
 
diff --git a/git-annex.cabal b/git-annex.cabal
index bd8c36063f..5a446ac7a8 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -904,6 +904,7 @@ Executable git-annex
     Messages.Internal
     Messages.JSON
     Messages.Progress
+    P2P.Address
     P2P.Auth
     P2P.IO
     P2P.Protocol

From a173f85578ee77bbc211e34cd6afd0bd1360eb76 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 29 Nov 2016 17:31:10 -0400
Subject: [PATCH 125/367] update docs for git-annex p2p command

It is not yet implemented.
---
 doc/git-annex-p2p.mdwn                      | 35 ++++++---------------
 doc/git-annex-remotedaemon.mdwn             |  3 ++
 doc/git-annex.mdwn                          |  5 +++
 doc/tips/peer_to_peer_network_with_tor.mdwn |  6 ++--
 4 files changed, 21 insertions(+), 28 deletions(-)

diff --git a/doc/git-annex-p2p.mdwn b/doc/git-annex-p2p.mdwn
index 41e1967ee6..8e06cc47c8 100644
--- a/doc/git-annex-p2p.mdwn
+++ b/doc/git-annex-p2p.mdwn
@@ -1,6 +1,6 @@
 # NAME
 
-git-annex p2p - manage peer-to-peer connections
+git-annex p2p - configure peer-2-peer links between repositories
 
 # SYNOPSIS
 
@@ -8,36 +8,21 @@ git annex p2p [options]
 
 # DESCRIPTION
 
-When using git-annex with peer-to-peer communication, this manages
-connections between the peers.
-
-Currently, git-annex supports peer-to-peer communication over Tor.
+This command can be used to link git-annex repositories over peer-2-peer
+networks.
 
 # OPTIONS
 
-* `--gen-address [name]`
+* `--gen-address`
 
-  Generates one or more addresses, which allow whoever knows them to access
-  your repository. The addresses are output on standard output, one per
-  supported P2P network.
+  Generates addresses that can be used to access this git-annex repository
+  over a P2P network. The address or addresses is output to stdout.
 
-  You can re-run this command repeatedly to generate as many addresses
-  as you like.
+* `--link-remote remotename address`
 
-  The name is an optional parameter, the name of the person or device you
-  intend to give this address to. Providig it makes it easier to use
-  `--remove-address`
-
-* `--link-remote name address`
-
-  Link the local repository to a remote repository. This sets up a git remote
-  with the specified name. The address is one generated by `--gen-address`
-  run on the remote repository.
-
-* `--remove-address [address|name]`
-
-  If you've given out an address to someone, and don't want to accept
-  connections from them anymore, this can be used to remove it.
+  Sets up a git remote with the specified remotename that is accessed over
+  a P2P network. The address is one generated in the remote repository using
+  `git annex p2p --gen-address`
 
 # SEE ALSO
 
diff --git a/doc/git-annex-remotedaemon.mdwn b/doc/git-annex-remotedaemon.mdwn
index 71dd32d309..d4960c4ff5 100644
--- a/doc/git-annex-remotedaemon.mdwn
+++ b/doc/git-annex-remotedaemon.mdwn
@@ -27,6 +27,7 @@ that supports it is 5.20140405.
 For tor-annex remotes, the remotedaemon runs as a tor hidden service,
 accepting connections from other nodes and serving up the contents of the
 repository. This is only done if you first run `git annex enable-tor`.
+Use `git annex p2p` to configure access to tor-annex remotes.
 
 # OPTIONS
 
@@ -48,6 +49,8 @@ comes back up.
 
 [[git-annex-enable-tor]](1)
 
+[[git-annex-p2p]](1)
+
 # AUTHOR
 
 Joey Hess <id@joeyh.name>
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 773e1b8176..d0cc310199 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -391,6 +391,11 @@ subdirectories).
 
   See [[git-annex-remotedaemon]](1) for details.
 
+* `p2p`
+
+  Configure peer-2-Peer links between repositories.
+
+  See [[git-annex-p2p]](1) for details.
 
 # QUERY COMMANDS
 
diff --git a/doc/tips/peer_to_peer_network_with_tor.mdwn b/doc/tips/peer_to_peer_network_with_tor.mdwn
index d7461a1e43..94470b96a2 100644
--- a/doc/tips/peer_to_peer_network_with_tor.mdwn
+++ b/doc/tips/peer_to_peer_network_with_tor.mdwn
@@ -16,7 +16,7 @@ To make git-annex use Tor, run these commands in your git-annex repository:
 	git annex remotedaemon
 	git annex p2p --gen-address
 
-The p2p-auth command will output a long address, such as:
+The p2p command will output a long address, such as:
 
 	tor-annex::eeaytkuhaupbarfi.onion:4412:7f53c5b65b8957ef626fd461ceaae8056e3dbc459ae715e4
 
@@ -55,8 +55,8 @@ You can run any commands you normally would to sync with that remote:
 
 	git annex sync --content peer1
 
-You can also generate an address for this new peer, by running
-`git annex p2p --gen`, and add that address to other peers using `git annex
+You can also generate an address for this new peer, by running `git annex
+p2p --gen-address`, and add that address to other peers using `git annex
 p2p --link-remote`. It's often useful to link peers up in both directions,
 so peer1 is a remote of peer2 and peer2 is a remote of peer1.
 

From f86a7f673c94f106e7a405332a4eb07843b34c63 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 29 Nov 2016 17:33:49 -0400
Subject: [PATCH 126/367] comments

---
 P2P/Address.hs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/P2P/Address.hs b/P2P/Address.hs
index 315219683f..862f06a9c3 100644
--- a/P2P/Address.hs
+++ b/P2P/Address.hs
@@ -51,10 +51,12 @@ instance FormatP2PAddress P2PAddressAuth where
 		authtoken <- toAuthToken (T.pack $ reverse ra)
 		return (P2PAddressAuth addr authtoken)
 
+-- | Load known P2P addresses for this repository.
 loadP2PAddresses :: Annex [P2PAddress]
 loadP2PAddresses = mapMaybe unformatP2PAddress . maybe [] lines
 	<$> readCacheCreds p2pAddressCredsFile
 
+-- | Store a new P2P address for this repository.
 storeP2PAddress :: P2PAddress -> Annex ()
 storeP2PAddress addr = do
 	addrs <- loadP2PAddresses

From 2d426903944ee57e358bc6d7bcf0a147bd746995 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 29 Nov 2016 17:36:03 -0400
Subject: [PATCH 127/367] devblog

---
 doc/devblog/day_430__tor_socket_problem.mdwn | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/devblog/day_430__tor_socket_problem.mdwn

diff --git a/doc/devblog/day_430__tor_socket_problem.mdwn b/doc/devblog/day_430__tor_socket_problem.mdwn
new file mode 100644
index 0000000000..7e7c8d1bd0
--- /dev/null
+++ b/doc/devblog/day_430__tor_socket_problem.mdwn
@@ -0,0 +1,13 @@
+Debian's tor daemon is very locked down in the directories it can read
+from, and so I've had a hard time finding a place to put the unix socket
+file for git-annex's tor hidden service. Painful details in 
+<http://bugs.debian.org/846275>. At least for now, I'm putting it under
+/etc/tor/, which is probably a FHS violation, but seems to be the only
+option that doesn't involve a lot of added complexity.
+
+--- 
+
+The Windows autobuilder is moving, since
+[NEST](http://nest-initiative.org/) is shutting down the server it has been
+using. Yury Zaytsev has set up a new Windows autobuilder, hosted at
+Dartmouth College this time.

From 53bf5cf8e60607be18d31ed903241fb9b368d5fa Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 29 Nov 2016 17:52:46 -0400
Subject: [PATCH 128/367] cleanup

---
 Utility/Tor.hs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Utility/Tor.hs b/Utility/Tor.hs
index e63bd82d4c..5457a5a24b 100644
--- a/Utility/Tor.hs
+++ b/Utility/Tor.hs
@@ -29,7 +29,6 @@ type UniqueIdent = String
 
 connectHiddenService :: OnionAddress -> OnionPort -> IO Socket
 connectHiddenService (OnionAddress address) port = do
-	hPutStrLn stderr $ show ("connect to", address, port)
 	(s, _) <- socksConnect torsockconf socksaddr
 	return s
   where
@@ -122,7 +121,7 @@ hiddenServiceHostnameFile uid ident = hiddenServiceDir uid ident </> "hostname"
 -- That should not be a problem if the UniqueIdent is around the length of
 -- a UUID.
 hiddenServiceSocketFile :: UserID -> UniqueIdent -> FilePath
-hiddenServiceSocketFile uid ident = etcDir </> "hidden_services" </> show uid ++ "_" ++ show ident </> "s"
+hiddenServiceSocketFile uid ident = etcDir </> "hidden_services" </> show uid ++ "_" ++ ident </> "s"
 
 -- | Sets up the directory for the socketFile, with appropriate
 -- permissions. Must run as root.

From f872ef25c5d1f0195b62063224fe7c1c24a3668f Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 29 Nov 2016 23:44:54 -0400
Subject: [PATCH 129/367] remove unused import

---
 Assistant/WebApp/Notifications.hs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Assistant/WebApp/Notifications.hs b/Assistant/WebApp/Notifications.hs
index 8d4a86cc7b..9d20773c09 100644
--- a/Assistant/WebApp/Notifications.hs
+++ b/Assistant/WebApp/Notifications.hs
@@ -16,7 +16,6 @@ import Assistant.DaemonStatus
 import Assistant.Types.Buddies
 import Utility.NotificationBroadcaster
 import Utility.Yesod
-import Utility.WebApp
 
 import Data.Text (Text)
 import qualified Data.Text as T

From 61a0be4fb0b1ea394b7615cbe87a55b388b42db4 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 30 Nov 2016 12:46:44 -0400
Subject: [PATCH 130/367] supposedly unused import was used

---
 Assistant/WebApp/Notifications.hs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Assistant/WebApp/Notifications.hs b/Assistant/WebApp/Notifications.hs
index 9d20773c09..8d4a86cc7b 100644
--- a/Assistant/WebApp/Notifications.hs
+++ b/Assistant/WebApp/Notifications.hs
@@ -16,6 +16,7 @@ import Assistant.DaemonStatus
 import Assistant.Types.Buddies
 import Utility.NotificationBroadcaster
 import Utility.Yesod
+import Utility.WebApp
 
 import Data.Text (Text)
 import qualified Data.Text as T

From 8354612131df127729545fcefde5e4f8aa0d7851 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 30 Nov 2016 12:50:49 -0400
Subject: [PATCH 131/367] prefer xdot over dot

* map: Run xdot if it's available in PATH. On OSX, the dot command
  does not support graphical display, while xdot does.
* Debian: xdot is a better interactive viewer than dot, so Suggest
  xdot, rather than graphviz.
---
 CHANGELOG              |  4 ++++
 Command/Map.hs         | 24 +++++++++++++++++-------
 debian/control         |  2 +-
 doc/git-annex-map.mdwn |  6 +++---
 4 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index 1e108d4a0b..76da79eaaf 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -4,6 +4,10 @@ git-annex (6.20161119) UNRELEASED; urgency=medium
     largerthan, mimetype, and smallerthan; the first two always failed
     to match, and the latter always matched.
   * Relicense 5 source files that are not part of the webapp from AGPL to GPL.
+  * map: Run xdot if it's available in PATH. On OSX, the dot command
+    does not support graphical display, while xdot does.
+  * Debian: xdot is a better interactive viewer than dot, so Suggest
+    xdot, rather than graphviz.
 
  -- Joey Hess <id@joeyh.name>  Mon, 21 Nov 2016 11:27:50 -0400
 
diff --git a/Command/Map.hs b/Command/Map.hs
index 2b21c40ba5..43c00d2572 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -47,15 +47,25 @@ start = do
 	liftIO $ writeFile file (drawMap rs trustmap umap)
 	next $ next $
 		ifM (Annex.getState Annex.fast)
-			( do
-				showLongNote $ "left map in " ++ file
-				return True
-			, do
-				showLongNote $ "running: dot -Tx11 " ++ file
-				showOutput
-				liftIO $ boolSystem "dot" [Param "-Tx11", File file]
+			( runViewer file []
+			, runViewer file
+	 			[ ("xdot", [File file])
+				, ("dot", [Param "-Tx11", File file])
+				]	
 			)
 
+runViewer :: FilePath -> [(String, [CommandParam])] -> Annex Bool
+runViewer file [] = do
+	showLongNote $ "left map in " ++ file
+	return True
+runViewer file ((c, ps):rest) = ifM (liftIO $ inPath c)
+	( do
+		showLongNote $ "running: " ++ c ++ unwords (toCommand ps)
+		showOutput
+		liftIO $ boolSystem c ps
+	, runViewer file rest
+	)
+
 {- Generates a graph for dot(1). Each repository, and any other uuids
  - (except for dead ones), are displayed as a node, and each of its
  - remotes is represented as an edge pointing at the node for the remote.
diff --git a/debian/control b/debian/control
index ec77a2946e..a07797462d 100644
--- a/debian/control
+++ b/debian/control
@@ -110,7 +110,7 @@ Recommends:
 	nocache,
 	aria2,
 Suggests:
-	graphviz,
+	xdot,
 	bup,
 	tahoe-lafs,
 	libnss-mdns,
diff --git a/doc/git-annex-map.mdwn b/doc/git-annex-map.mdwn
index cf28a958e0..ece26b3672 100644
--- a/doc/git-annex-map.mdwn
+++ b/doc/git-annex-map.mdwn
@@ -10,8 +10,8 @@ git annex map
 
 Helps you keep track of your repositories, and the connections between them,
 by going out and looking at all the ones it can get to, and generating a
-Graphviz file displaying it all. If the `dot` command is available, it is
-used to display the file to your screen (using x11 backend).
+Graphviz file displaying it all. If the `xdot` or `dot` command is available,
+it is used to display the file to your screen.
   
 This command only connects to hosts that the host it's run on can
 directly connect to. It does not try to tunnel through intermediate hosts.
@@ -37,7 +37,7 @@ on that host.
 
 * `--fast`
 
-  Disable using `dot` to display the generated Graphviz file.
+  Don't display the generated Graphviz file, but save it for later use.
 
 # SEE ALSO
 

From 568d81944a5ae07cfc39b06aeede69f7c44c9c4c Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 30 Nov 2016 14:16:57 -0400
Subject: [PATCH 132/367] avoid too-long command synopsis

It was making git-annex usage output columns far too wide
---
 Command/Reinject.hs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Command/Reinject.hs b/Command/Reinject.hs
index 97aa602e7b..7d2da94202 100644
--- a/Command/Reinject.hs
+++ b/Command/Reinject.hs
@@ -16,8 +16,7 @@ import Types.KeySource
 cmd :: Command
 cmd = command "reinject" SectionUtility 
 	"inject content of file back into annex"
-	(paramRepeating (paramPair "SRC" "DEST")
-		`paramOr` "--known " ++ paramRepeating "SRC")
+	(paramRepeating (paramPair "SRC" "DEST"))
 	(seek <$$> optParser)
 
 data ReinjectOptions = ReinjectOptions

From ac0cb5c2cc2fdbb0b0632965ed241d59dfdc2223 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 30 Nov 2016 14:19:26 -0400
Subject: [PATCH 133/367] max authtoken length is 128

It was stopping at 128, so the 512 was only incorrect, it didn't change
behavior.
---
 Assistant/Threads/WebApp.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 576feb5f01..a5cd385046 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -76,7 +76,7 @@ webAppThread assistantdata urlrenderer noannex cannotrun postfirstrun listenhost
 #endif
 	webapp <- WebApp
 		<$> pure assistantdata
-		<*> genAuthToken 512
+		<*> genAuthToken 128
 		<*> getreldir
 		<*> pure staticRoutes
 		<*> pure postfirstrun

From bfc83058140d94c2595623eb30f95a44b986521a Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 30 Nov 2016 14:35:24 -0400
Subject: [PATCH 134/367] implement p2p command

---
 Annex/SpecialRemote.hs                      |  3 +-
 CHANGELOG                                   |  1 +
 CmdLine/GitAnnex.hs                         |  2 +
 Command/EnableRemote.hs                     |  7 +--
 Command/P2P.hs                              | 61 +++++++++++++++++++++
 P2P/Address.hs                              |  5 +-
 P2P/Auth.hs                                 | 37 +++++++------
 doc/git-annex-p2p.mdwn                      |  8 ++-
 doc/tips/peer_to_peer_network_with_tor.mdwn | 16 +++---
 git-annex.cabal                             |  3 +
 10 files changed, 110 insertions(+), 33 deletions(-)
 create mode 100644 Command/P2P.hs

diff --git a/Annex/SpecialRemote.hs b/Annex/SpecialRemote.hs
index 02799db854..0fd24f0238 100644
--- a/Annex/SpecialRemote.hs
+++ b/Annex/SpecialRemote.hs
@@ -13,12 +13,11 @@ import Types.Remote (RemoteConfig, RemoteConfigKey, typename, setup)
 import Logs.Remote
 import Logs.Trust
 import qualified Git.Config
+import Git.Types (RemoteName)
 
 import qualified Data.Map as M
 import Data.Ord
 
-type RemoteName = String
-
 {- See if there's an existing special remote with this name.
  -
  - Prefer remotes that are not dead when a name appears multiple times. -}
diff --git a/CHANGELOG b/CHANGELOG
index 2eef5a4226..b532e7674b 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,7 @@
 git-annex (6.20161119) UNRELEASED; urgency=medium
 
   * enable-tor: New command, enables tor hidden service for P2P syncing.
+  * p2p: New command, allows linking repositories using a P2P network.
   * remotedaemon: Serve tor hidden service.
   * Added git-remote-tor-annex, which allows git pull and push to the tor
     hidden service.
diff --git a/CmdLine/GitAnnex.hs b/CmdLine/GitAnnex.hs
index 0fa14c98bb..a12366b742 100644
--- a/CmdLine/GitAnnex.hs
+++ b/CmdLine/GitAnnex.hs
@@ -96,6 +96,7 @@ import qualified Command.Direct
 import qualified Command.Indirect
 import qualified Command.Upgrade
 import qualified Command.Forget
+import qualified Command.P2P
 import qualified Command.Proxy
 import qualified Command.DiffDriver
 import qualified Command.Smudge
@@ -204,6 +205,7 @@ cmds testoptparser testrunner =
 	, Command.Indirect.cmd
 	, Command.Upgrade.cmd
 	, Command.Forget.cmd
+	, Command.P2P.cmd
 	, Command.Proxy.cmd
 	, Command.DiffDriver.cmd
 	, Command.Smudge.cmd
diff --git a/Command/EnableRemote.hs b/Command/EnableRemote.hs
index e1af8bb7a4..61cd543e6f 100644
--- a/Command/EnableRemote.hs
+++ b/Command/EnableRemote.hs
@@ -12,6 +12,7 @@ import qualified Annex
 import qualified Logs.Remote
 import qualified Types.Remote as R
 import qualified Git
+import qualified Git.Types as Git
 import qualified Annex.SpecialRemote
 import qualified Remote
 import qualified Types.Remote as Remote
@@ -40,9 +41,7 @@ start (name:rest) = go =<< filter matchingname <$> Annex.fromRepo Git.remotes
 		=<< Annex.SpecialRemote.findExisting name
 	go (r:_) = startNormalRemote name r
 
-type RemoteName = String
-
-startNormalRemote :: RemoteName -> Git.Repo -> CommandStart
+startNormalRemote :: Git.RemoteName -> Git.Repo -> CommandStart
 startNormalRemote name r = do
 	showStart "enableremote" name
 	next $ next $ do
@@ -51,7 +50,7 @@ startNormalRemote name r = do
 		u <- getRepoUUID r'
 		return $ u /= NoUUID
 
-startSpecialRemote :: RemoteName -> Remote.RemoteConfig -> Maybe (UUID, Remote.RemoteConfig) -> CommandStart
+startSpecialRemote :: Git.RemoteName -> Remote.RemoteConfig -> Maybe (UUID, Remote.RemoteConfig) -> CommandStart
 startSpecialRemote name config Nothing = do
 	m <- Annex.SpecialRemote.specialRemoteMap
 	confm <- Logs.Remote.readRemoteLog
diff --git a/Command/P2P.hs b/Command/P2P.hs
new file mode 100644
index 0000000000..ec6e4be96a
--- /dev/null
+++ b/Command/P2P.hs
@@ -0,0 +1,61 @@
+{- git-annex command
+ -
+ - Copyright 2016 Joey Hess <id@joeyh.name>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.P2P where
+
+import Command
+import Git.Types
+import P2P.Address
+import P2P.Auth
+import Utility.AuthToken
+
+cmd :: Command
+cmd = command "p2p" SectionSetup
+	"configure peer-2-peer links between repositories"
+	paramNothing (seek <$$> optParser)
+
+data P2POpts
+	= GenAddresses
+	| LinkRemote P2PAddressAuth RemoteName
+
+optParser :: CmdParamsDesc -> Parser P2POpts 
+optParser _ = genaddresses <|> linkremote
+  where
+	genaddresses = flag' GenAddresses
+		( long "gen-addresses"
+		<> help "generate addresses that allow accessing this repository over P2P networks"
+		)
+	linkremote = LinkRemote
+		<$> option readaddr
+			( long "link"
+			<> metavar paramAddress
+			<> help "address of the peer to link with"
+			)
+		<*> strOption
+			( long "named"
+			<> metavar paramRemote
+			<> help "specify name to use for git remote"
+			)
+	readaddr = eitherReader $ maybe (Left "address parse error") Right
+		. unformatP2PAddress
+
+seek :: P2POpts -> CommandSeek
+seek GenAddresses = do
+	addrs <- loadP2PAddresses
+	if null addrs
+		then giveup "No P2P networks are currrently available."
+		else do
+			authtoken <- liftIO $ genAuthToken 128
+			storeP2PAuthToken authtoken
+			-- Only addresses are output to stdout, to allow
+			-- scripting.
+			earlyWarning "These addresses allow access to this git-annex repository. Only share them with people you trust with that access, using trusted communication channels!"
+			liftIO $ putStr $ unlines $
+				map formatP2PAddress $
+				map (`P2PAddressAuth` authtoken) addrs
+seek (LinkRemote addr name) = do
+	
diff --git a/P2P/Address.hs b/P2P/Address.hs
index 862f06a9c3..19ff82a89b 100644
--- a/P2P/Address.hs
+++ b/P2P/Address.hs
@@ -23,7 +23,10 @@ import qualified Data.Text as T
 data P2PAddress = TorAnnex OnionAddress OnionPort
 	deriving (Eq, Show)
 
--- | A P2P address, with an AuthToken
+-- | A P2P address, with an AuthToken.
+--
+-- This is enough information to connect to the peer, and authenticate with
+-- it.
 data P2PAddressAuth = P2PAddressAuth P2PAddress AuthToken
 	deriving (Eq, Show)
 
diff --git a/P2P/Auth.hs b/P2P/Auth.hs
index 5c3feb7132..2482c1dc0b 100644
--- a/P2P/Auth.hs
+++ b/P2P/Auth.hs
@@ -1,4 +1,4 @@
-{- P2P protocol, authorization
+{- P2P authtokens
  -
  - Copyright 2016 Joey Hess <id@joeyh.name>
  -
@@ -7,24 +7,29 @@
 
 module P2P.Auth where
 
-import Common
+import Annex.Common
+import Creds
 import Utility.AuthToken
 
 import qualified Data.Text as T
 
--- Use .git/annex/creds/p2p to hold AuthTokens of authorized peers.
-getAuthTokens :: Annex AllowedAuthTokens
-getAuthTokens = allowedAuthTokens <$> getAuthTokens'
+-- | Load authtokens that are accepted by this repository.
+loadP2PAuthTokens :: Annex AllowedAuthTokens
+loadP2PAuthTokens = allowedAuthTokens <$> loadP2PAuthTokens'
 
-getAuthTokens' :: Annex [AuthTokens]
-getAuthTokens' = mapMaybe toAuthToken
-	. map T.pack
-	. lines
-	. fromMaybe []
-	<$> readCacheCreds "tor"
+loadP2PAuthTokens' :: Annex [AuthToken]
+loadP2PAuthTokens' = mapMaybe toAuthToken
+        . map T.pack
+        . lines
+        . fromMaybe []
+        <$> readCacheCreds p2pAuthCredsFile
 
-addAuthToken :: AuthToken -> Annex ()
-addAuthToken t = do
-	ts <- getAuthTokens'
-	let d = unlines $ map (T.unpack . fromAuthToken) (t:ts)
-	writeCacheCreds d "tor"
+storeP2PAuthToken :: AuthToken -> Annex ()
+storeP2PAuthToken t = do
+	ts <- loadP2PAuthTokens'
+	unless (t `elem` ts) $ do
+		let d = unlines $ map (T.unpack . fromAuthToken) (t:ts)
+		writeCacheCreds d p2pAuthCredsFile
+
+p2pAuthCredsFile :: FilePath
+p2pAuthCredsFile = "p2pauth"
diff --git a/doc/git-annex-p2p.mdwn b/doc/git-annex-p2p.mdwn
index 8e06cc47c8..049f900145 100644
--- a/doc/git-annex-p2p.mdwn
+++ b/doc/git-annex-p2p.mdwn
@@ -11,14 +11,18 @@ git annex p2p [options]
 This command can be used to link git-annex repositories over peer-2-peer
 networks.
 
+Currently, the only P2P network supported by git-annex is Tor hidden
+services.
+
 # OPTIONS
 
 * `--gen-address`
 
   Generates addresses that can be used to access this git-annex repository
-  over a P2P network. The address or addresses is output to stdout.
+  over the available P2P networks. The address or addresses is output to
+  stdout.
 
-* `--link-remote remotename address`
+* `--link address --named remotename`
 
   Sets up a git remote with the specified remotename that is accessed over
   a P2P network. The address is one generated in the remote repository using
diff --git a/doc/tips/peer_to_peer_network_with_tor.mdwn b/doc/tips/peer_to_peer_network_with_tor.mdwn
index 94470b96a2..de018e3ced 100644
--- a/doc/tips/peer_to_peer_network_with_tor.mdwn
+++ b/doc/tips/peer_to_peer_network_with_tor.mdwn
@@ -44,7 +44,7 @@ repository:
 
 Now, tell the new peer about the address of the first peer:
 
-	git annex p2p --link-remote peer1 tor-annnex::eeaytkuhaupbarfi.onion:4412:7f53c5b65b8957ef626fd461ceaae8056e3dbc459ae715e4
+	git annex p2p --link tor-annnex::eeaytkuhaupbarfi.onion:4412:7f53c5b65b8957ef626fd461ceaae8056e3dbc459ae715e4 --named peer1
 
 (Of course, you should paste in the address you generated earlier, 
 not the example one shown above.)
@@ -56,8 +56,8 @@ You can run any commands you normally would to sync with that remote:
 	git annex sync --content peer1
 
 You can also generate an address for this new peer, by running `git annex
-p2p --gen-address`, and add that address to other peers using `git annex
-p2p --link-remote`. It's often useful to link peers up in both directions,
+p2p --gen-address`, and link other peers to that address using `git annex
+p2p --link`. It's often useful to link peers up in both directions,
 so peer1 is a remote of peer2 and peer2 is a remote of peer1.
 
 Any number of peers can be connected this way, within reason.
@@ -88,14 +88,14 @@ You can `git pull`, push, etc with those onion addresses:
 	git remote add peer1 tor-annnex::eeaytkuhaupbarfi.onion:4412
 
 Onion addresses are semi-public. When you add a remote, they appear in your
-`.git/config` file. So, there's a second level of authentication that
-git-annex uses to make sure that only people you want to can access your
-repository over Tor. That takes the form of a long string of numbers and
-letters, like "7f53c5b65b8957ef626fd461ceaae8056e3dbc459ae715e4".
+`.git/config` file. For security, there's a second level of authentication
+that git-annex uses to make sure that only people you want to can access
+your repository over Tor. That takes the form of a long string of numbers
+and letters, like "7f53c5b65b8957ef626fd461ceaae8056e3dbc459ae715e4".
 
 The addresses generated by `git annex peer --gen-address` 
 combine the onion address with the authentication data.
 
-When you run `git annex peer --link-remote`, it sets up a git remote using
+When you run `git annex peer --link`, it sets up a git remote using
 the onion address, and it stashes the authentication data away in a file in
 `.git/annex/creds/`
diff --git a/git-annex.cabal b/git-annex.cabal
index 5a446ac7a8..6991d2a048 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -91,6 +91,7 @@ Extra-Source-Files:
   doc/git-annex-mirror.mdwn
   doc/git-annex-move.mdwn
   doc/git-annex-numcopies.mdwn
+  doc/git-annex-p2p.mdwn
   doc/git-annex-pre-commit.mdwn
   doc/git-annex-preferred-content.mdwn
   doc/git-annex-proxy.mdwn
@@ -727,6 +728,7 @@ Executable git-annex
     Command.DropKey
     Command.DropUnused
     Command.EnableRemote
+    Command.EnableTor
     Command.ExamineKey
     Command.Expire
     Command.Find
@@ -762,6 +764,7 @@ Executable git-annex
     Command.Move
     Command.NotifyChanges
     Command.NumCopies
+    Command.P2P
     Command.PreCommit
     Command.Proxy
     Command.ReKey

From a26a0eb6187685f4750e744a5c6c5a5101db96fe Mon Sep 17 00:00:00 2001
From: "pablo@4f5b00d7b40f0a52e51761da420d6ff8e7c9aca7" <pablo@web>
Date: Wed, 30 Nov 2016 18:56:32 +0000
Subject: [PATCH 135/367]

---
 doc/bugs/wget_always_shows_progress_bar.mdwn | 24 ++++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 doc/bugs/wget_always_shows_progress_bar.mdwn

diff --git a/doc/bugs/wget_always_shows_progress_bar.mdwn b/doc/bugs/wget_always_shows_progress_bar.mdwn
new file mode 100644
index 0000000000..2829b39b0e
--- /dev/null
+++ b/doc/bugs/wget_always_shows_progress_bar.mdwn
@@ -0,0 +1,24 @@
+### Please describe the problem.
+
+git annex addurl or importfeed operations were quiet on git-annex versions older than 5.20141219, if
+annex.web-options was set to "--quiet". But now the --show-progress option is always passed to wget.
+
+In some use cases this might be nice, but I'm using GNU Parallel to update multiple podcast feeds
+concurrently, and it causes wget to output the ugly "dot" indicator for every feed, which is totally
+useless since parallel buffers and groups the output. Adding "--no-show-progress" to web-options
+does not help (it does not override --show-progress), nor does redirecting stdout to /dev/null.
+Redirecting stderr would hide possible errors.
+
+### What steps will reproduce the problem?
+
+parallel git annex importfeed --relaxed --quiet ::: http://feeds.feedburner.com/freakonomicsradio
+
+### What version of git-annex are you using? On what operating system?
+
+5.20151208-1~bpo8+1 on Debian.
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+I love git annex and use it daily.
+
+

From 3ab12ba9230284d3a4a36bcb5fd85bc2e53ecfd0 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 30 Nov 2016 15:14:54 -0400
Subject: [PATCH 136/367] implement p2p --link

This commit was sponsored by Riku Voipio.
---
 Command/P2P.hs                              | 81 +++++++++++++--------
 P2P/Auth.hs                                 | 20 +++++
 doc/git-annex-p2p.mdwn                      |  8 +-
 doc/tips/peer_to_peer_network_with_tor.mdwn | 15 ++--
 4 files changed, 84 insertions(+), 40 deletions(-)

diff --git a/Command/P2P.hs b/Command/P2P.hs
index ec6e4be96a..e2a7ab85d2 100644
--- a/Command/P2P.hs
+++ b/Command/P2P.hs
@@ -8,10 +8,12 @@
 module Command.P2P where
 
 import Command
-import Git.Types
 import P2P.Address
 import P2P.Auth
 import Utility.AuthToken
+import Git.Types
+import qualified Git.Remote
+import qualified Git.Command
 
 cmd :: Command
 cmd = command "p2p" SectionSetup
@@ -20,7 +22,7 @@ cmd = command "p2p" SectionSetup
 
 data P2POpts
 	= GenAddresses
-	| LinkRemote P2PAddressAuth RemoteName
+	| LinkRemote RemoteName
 
 optParser :: CmdParamsDesc -> Parser P2POpts 
 optParser _ = genaddresses <|> linkremote
@@ -29,33 +31,52 @@ optParser _ = genaddresses <|> linkremote
 		( long "gen-addresses"
 		<> help "generate addresses that allow accessing this repository over P2P networks"
 		)
-	linkremote = LinkRemote
-		<$> option readaddr
-			( long "link"
-			<> metavar paramAddress
-			<> help "address of the peer to link with"
-			)
-		<*> strOption
-			( long "named"
-			<> metavar paramRemote
-			<> help "specify name to use for git remote"
-			)
-	readaddr = eitherReader $ maybe (Left "address parse error") Right
-		. unformatP2PAddress
+	linkremote = LinkRemote <$> strOption
+		( long "link"
+		<> metavar paramRemote
+		<> help "specify name to use for git remote"
+		)
 
 seek :: P2POpts -> CommandSeek
-seek GenAddresses = do
-	addrs <- loadP2PAddresses
-	if null addrs
-		then giveup "No P2P networks are currrently available."
-		else do
-			authtoken <- liftIO $ genAuthToken 128
-			storeP2PAuthToken authtoken
-			-- Only addresses are output to stdout, to allow
-			-- scripting.
-			earlyWarning "These addresses allow access to this git-annex repository. Only share them with people you trust with that access, using trusted communication channels!"
-			liftIO $ putStr $ unlines $
-				map formatP2PAddress $
-				map (`P2PAddressAuth` authtoken) addrs
-seek (LinkRemote addr name) = do
-	
+seek GenAddresses = genAddresses =<< loadP2PAddresses
+seek (LinkRemote name) = commandAction $
+	linkRemote (Git.Remote.makeLegalName name)
+
+-- Only addresses are output to stdout, to allow scripting.
+genAddresses :: [P2PAddress] -> Annex ()
+genAddresses [] = giveup "No P2P networks are currrently available."
+genAddresses addrs = do
+	authtoken <- liftIO $ genAuthToken 128
+	storeP2PAuthToken authtoken
+	earlyWarning "These addresses allow access to this git-annex repository. Only share them with people you trust with that access, using trusted communication channels!"
+	liftIO $ putStr $ unlines $
+		map formatP2PAddress $
+			map (`P2PAddressAuth` authtoken) addrs
+
+-- Address is read from stdin, to avoid leaking it in shell history.
+linkRemote :: RemoteName -> CommandStart
+linkRemote remotename = do
+	showStart "p2p link" remotename
+	next $ next prompt
+  where
+	prompt = do
+		liftIO $ putStrLn ""
+		liftIO $ putStr "Enter address: "
+		liftIO $ hFlush stdout
+		s <- liftIO getLine
+		if null s
+			then do
+				liftIO $ hPutStrLn stderr "Nothing entered, giving up."
+				return False
+			else case unformatP2PAddress s of
+				Nothing -> do
+					liftIO $ hPutStrLn stderr "Unable to parse that address, please check its format and try again."
+					prompt
+				Just addr -> setup addr
+	setup (P2PAddressAuth addr authtoken) = do
+		storeP2PRemoteAuthToken addr authtoken
+		inRepo $ Git.Command.runBool
+			[ Param "remote", Param "add"
+			, Param remotename
+			, Param (formatP2PAddress addr)
+			]
diff --git a/P2P/Auth.hs b/P2P/Auth.hs
index 2482c1dc0b..2c84659005 100644
--- a/P2P/Auth.hs
+++ b/P2P/Auth.hs
@@ -9,7 +9,9 @@ module P2P.Auth where
 
 import Annex.Common
 import Creds
+import P2P.Address
 import Utility.AuthToken
+import Utility.Tor
 
 import qualified Data.Text as T
 
@@ -24,6 +26,7 @@ loadP2PAuthTokens' = mapMaybe toAuthToken
         . fromMaybe []
         <$> readCacheCreds p2pAuthCredsFile
 
+-- | Stores an AuthToken, making it be accepted by this repository.
 storeP2PAuthToken :: AuthToken -> Annex ()
 storeP2PAuthToken t = do
 	ts <- loadP2PAuthTokens'
@@ -33,3 +36,20 @@ storeP2PAuthToken t = do
 
 p2pAuthCredsFile :: FilePath
 p2pAuthCredsFile = "p2pauth"
+
+-- | Loads the AuthToken to use when connecting with a given P2P address.
+loadP2PRemoteAuthToken :: P2PAddress -> Annex (Maybe AuthToken)
+loadP2PRemoteAuthToken addr = maybe Nothing (toAuthToken . T.pack)
+	<$> readCacheCreds (addressCredsFile addr)
+
+-- | Stores the AuthToken o use when connecting with a given P2P address.
+storeP2PRemoteAuthToken :: P2PAddress -> AuthToken -> Annex ()
+storeP2PRemoteAuthToken addr t = writeCacheCreds
+	(T.unpack $ fromAuthToken t)
+	(addressCredsFile addr)
+
+addressCredsFile :: P2PAddress -> FilePath
+-- We can omit the port and just use the onion address for the creds file,
+-- because any given tor hidden service runs on a single port and has a
+-- unique onion address.
+addressCredsFile (TorAnnex (OnionAddress onionaddr) _port) = onionaddr
diff --git a/doc/git-annex-p2p.mdwn b/doc/git-annex-p2p.mdwn
index 049f900145..5bf48178fe 100644
--- a/doc/git-annex-p2p.mdwn
+++ b/doc/git-annex-p2p.mdwn
@@ -22,11 +22,13 @@ services.
   over the available P2P networks. The address or addresses is output to
   stdout.
 
-* `--link address --named remotename`
+* `--link remotename`
 
   Sets up a git remote with the specified remotename that is accessed over
-  a P2P network. The address is one generated in the remote repository using
-  `git annex p2p --gen-address`
+  a P2P network. 
+  
+  This will prompt for an address to be entered; you should paste in the
+  address that was generated by --gen-address in the remote repository.
 
 # SEE ALSO
 
diff --git a/doc/tips/peer_to_peer_network_with_tor.mdwn b/doc/tips/peer_to_peer_network_with_tor.mdwn
index de018e3ced..0481874587 100644
--- a/doc/tips/peer_to_peer_network_with_tor.mdwn
+++ b/doc/tips/peer_to_peer_network_with_tor.mdwn
@@ -42,16 +42,17 @@ repository:
 	sudo git annex enable-tor
 	git annex remotedaemon
 
-Now, tell the new peer about the address of the first peer:
+Now, tell the new peer about the address of the first peer.
+This will make a git remote named "peer1", which connects,
+through Tor, to the repository on the other peer.
 
-	git annex p2p --link tor-annnex::eeaytkuhaupbarfi.onion:4412:7f53c5b65b8957ef626fd461ceaae8056e3dbc459ae715e4 --named peer1
+	git annex p2p --link peer1
 
-(Of course, you should paste in the address you generated earlier, 
-not the example one shown above.)
+That command will prompt for an address; paste in the address that was
+generated on the first peer, and then press Enter.
 
-Now this git-annex repository will have a remote named "peer1"
-which connects, through Tor, to the repository on the other peer.
-You can run any commands you normally would to sync with that remote:
+Now you can run any commands you normally would to sync with the
+peer1 remote:
 
 	git annex sync --content peer1
 

From b88e44ea9ab935e7915e9441bab30b161cfacb93 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 30 Nov 2016 15:26:16 -0400
Subject: [PATCH 137/367] use P2P auth for git-remote-tor-annex

This changes the environment variable name to the more generic
GIT_ANNEX_P2P_AUTHTOKEN.

This commit was sponsored by andrea rota.
---
 CmdLine/GitRemoteTorAnnex.hs  |  6 ++++--
 P2P/Auth.hs                   | 15 +++++++++++++--
 Remote/Helper/Tor.hs          | 18 ------------------
 doc/git-remote-tor-annex.mdwn |  2 +-
 4 files changed, 18 insertions(+), 23 deletions(-)

diff --git a/CmdLine/GitRemoteTorAnnex.hs b/CmdLine/GitRemoteTorAnnex.hs
index 3b2dcc0509..ea4532ae63 100644
--- a/CmdLine/GitRemoteTorAnnex.hs
+++ b/CmdLine/GitRemoteTorAnnex.hs
@@ -16,6 +16,8 @@ import Remote.Helper.Tor
 import Utility.Tor
 import Utility.AuthToken
 import Annex.UUID
+import P2P.Address
+import P2P.Auth
 
 run :: [String] -> IO ()
 run (_remotename:address:[]) = forever $ do
@@ -54,7 +56,7 @@ connectService address port service = do
 	state <- Annex.new =<< Git.CurrentRepo.get
 	Annex.eval state $ do
 		authtoken <- fromMaybe nullAuthToken
-			<$> getTorAuthTokenFor address
+			<$> loadP2PRemoteAuthToken (TorAnnex address port)
 		myuuid <- getUUID
 		g <- Annex.gitRepo
 		h <- liftIO $ torHandle =<< connectHiddenService address port
@@ -62,4 +64,4 @@ connectService address port service = do
 			v <- auth myuuid authtoken
 			case v of
 				Just _theiruuid -> connect service stdin stdout
-				Nothing -> giveup $ "authentication failed, perhaps you need to set " ++ torAuthTokenEnv
+				Nothing -> giveup $ "authentication failed, perhaps you need to set " ++ p2pAuthTokenEnv
diff --git a/P2P/Auth.hs b/P2P/Auth.hs
index 2c84659005..0025957c75 100644
--- a/P2P/Auth.hs
+++ b/P2P/Auth.hs
@@ -12,6 +12,7 @@ import Creds
 import P2P.Address
 import Utility.AuthToken
 import Utility.Tor
+import Utility.Env
 
 import qualified Data.Text as T
 
@@ -38,9 +39,19 @@ p2pAuthCredsFile :: FilePath
 p2pAuthCredsFile = "p2pauth"
 
 -- | Loads the AuthToken to use when connecting with a given P2P address.
+--
+-- It's loaded from the first line of the creds file, but
+-- GIT_ANNEX_P2P_AUTHTOKEN overrides.
 loadP2PRemoteAuthToken :: P2PAddress -> Annex (Maybe AuthToken)
-loadP2PRemoteAuthToken addr = maybe Nothing (toAuthToken . T.pack)
-	<$> readCacheCreds (addressCredsFile addr)
+loadP2PRemoteAuthToken addr = maybe Nothing mk <$> getM id
+	[ liftIO $ getEnv "GIT_ANNEX_P2P_AUTHTOKEN"
+	, readCacheCreds (addressCredsFile addr)
+	]
+  where
+	mk = toAuthToken . T.pack . takeWhile (/= '\n')
+
+p2pAuthTokenEnv :: String
+p2pAuthTokenEnv = "GIT_ANNEX_P2P_AUTHTOKEN"
 
 -- | Stores the AuthToken o use when connecting with a given P2P address.
 storeP2PRemoteAuthToken :: P2PAddress -> AuthToken -> Annex ()
diff --git a/Remote/Helper/Tor.hs b/Remote/Helper/Tor.hs
index 25d1920235..b5a819c3b7 100644
--- a/Remote/Helper/Tor.hs
+++ b/Remote/Helper/Tor.hs
@@ -8,26 +8,8 @@
 module Remote.Helper.Tor where
 
 import Annex.Common
-import Utility.AuthToken
-import Creds
-import Utility.Tor
-import Utility.Env
 
 import Network.Socket
-import qualified Data.Text as T
-
--- Read the first line of the creds file. Environment variable overrides.
-getTorAuthTokenFor :: OnionAddress -> Annex (Maybe AuthToken)
-getTorAuthTokenFor (OnionAddress onionaddress) =
-	maybe Nothing mk <$> getM id 
-		[ liftIO $ getEnv torAuthTokenEnv
-		, readCacheCreds onionaddress
-		]
-  where
-	mk = toAuthToken . T.pack . takeWhile (/= '\n')
-
-torAuthTokenEnv :: String
-torAuthTokenEnv = "GIT_ANNEX_TOR_AUTHTOKEN"
 
 torHandle :: Socket -> IO Handle
 torHandle s = do
diff --git a/doc/git-remote-tor-annex.mdwn b/doc/git-remote-tor-annex.mdwn
index 63b459ed8e..4e41de877b 100644
--- a/doc/git-remote-tor-annex.mdwn
+++ b/doc/git-remote-tor-annex.mdwn
@@ -15,7 +15,7 @@ over tor(1), communicating with a tor hidden service.
 
 The tor hidden service probably requires an authtoken to use it.
 The authtoken can be provided in the environment variable
-`GIT_ANNEX_TOR_AUTHTOKEN`. Or, if there is a file in 
+`GIT_ANNEX_P2P_AUTHTOKEN`. Or, if there is a file in 
 `.git/annex/creds/` matching the onion address of the hidden
 service, its first line is used as the authtoken.
 

From e714e0f67a4c706c3a99f542a7ea33dc3ba9cf43 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 30 Nov 2016 16:38:16 -0400
Subject: [PATCH 138/367] actually check p2p authtokens for tor connections

This commit was sponsored by Ethan Aubin.
---
 CmdLine/GitRemoteTorAnnex.hs  |  8 +++++-
 P2P/IO.hs                     | 51 ++++++++++++++++++-----------------
 RemoteDaemon/Transport/Tor.hs | 19 ++++++++++---
 3 files changed, 49 insertions(+), 29 deletions(-)

diff --git a/CmdLine/GitRemoteTorAnnex.hs b/CmdLine/GitRemoteTorAnnex.hs
index ea4532ae63..72211c995c 100644
--- a/CmdLine/GitRemoteTorAnnex.hs
+++ b/CmdLine/GitRemoteTorAnnex.hs
@@ -60,7 +60,13 @@ connectService address port service = do
 		myuuid <- getUUID
 		g <- Annex.gitRepo
 		h <- liftIO $ torHandle =<< connectHiddenService address port
-		runNetProtoHandle h h g $ do
+		let runenv = RunEnv
+			{ runRepo = g
+			, runCheckAuth = const False
+			, runIhdl = h
+			, runOhdl = h
+			}
+		runNetProtoHandle runenv $ do
 			v <- auth myuuid authtoken
 			case v of
 				Just _theiruuid -> connect service stdin stdout
diff --git a/P2P/IO.hs b/P2P/IO.hs
index 822eb524ef..c0b14edca7 100644
--- a/P2P/IO.hs
+++ b/P2P/IO.hs
@@ -5,17 +5,19 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-{-# LANGUAGE RankNTypes, CPP #-}
+{-# LANGUAGE RankNTypes, ScopedTypeVariables, CPP #-}
 
 module P2P.IO
-	( RunProto
+	( RunEnv(..)
 	, runNetProtoHandle
+	, runNetHandle
 	) where
 
 import P2P.Protocol
 import Utility.Process
 import Git
 import Git.Command
+import Utility.AuthToken
 import Utility.SafeCommand
 import Utility.SimpleProtocol
 import Utility.Exception
@@ -32,33 +34,34 @@ import qualified Data.ByteString.Lazy as L
 
 type RunProto = forall a m. (MonadIO m, MonadMask m) => Proto a -> m (Maybe a)
 
-data S = S
-	{ repo :: Repo
-	, ihdl :: Handle
-	, ohdl :: Handle
+data RunEnv = RunEnv
+	{ runRepo :: Repo
+	, runCheckAuth :: (AuthToken -> Bool)
+	, runIhdl :: Handle
+	, runOhdl :: Handle
 	}
 
 -- Implementation of the protocol, communicating with a peer
 -- over a Handle. No Local actions will be run.
-runNetProtoHandle :: (MonadIO m, MonadMask m) => Handle -> Handle -> Repo -> Proto a -> m (Maybe a)
-runNetProtoHandle i o r = go
+runNetProtoHandle :: (MonadIO m, MonadMask m) => RunEnv -> Proto a -> m (Maybe a)
+runNetProtoHandle runenv = go
   where
 	go :: RunProto
 	go (Pure v) = pure (Just v)
-	go (Free (Net n)) = runNetHandle (S r i o) go n
+	go (Free (Net n)) = runNetHandle runenv go n
 	go (Free (Local _)) = return Nothing
 
-runNetHandle :: (MonadIO m, MonadMask m) => S -> RunProto -> NetF (Proto a) -> m (Maybe a)
-runNetHandle s runner f = case f of
+runNetHandle :: (MonadIO m, MonadMask m) => RunEnv -> RunProto -> NetF (Proto a) -> m (Maybe a)
+runNetHandle runenv runner f = case f of
 	SendMessage m next -> do
 		v <- liftIO $ tryIO $ do
-			hPutStrLn (ohdl s) (unwords (formatMessage m))
-			hFlush (ohdl s)
+			hPutStrLn (runOhdl runenv) (unwords (formatMessage m))
+			hFlush (runOhdl runenv)
 		case v of
 			Left _e -> return Nothing
 			Right () -> runner next
 	ReceiveMessage next -> do
-		v <- liftIO $ tryIO $ hGetLine (ihdl s)
+		v <- liftIO $ tryIO $ hGetLine (runIhdl runenv)
 		case v of
 			Left _e -> return Nothing
 			Right l -> case parseMessage l of
@@ -69,18 +72,18 @@ runNetHandle s runner f = case f of
 					next e
 	SendBytes _len b next -> do
 		v <- liftIO $ tryIO $ do
-			L.hPut (ohdl s) b
-			hFlush (ohdl s)
+			L.hPut (runOhdl runenv) b
+			hFlush (runOhdl runenv)
 		case v of
 			Left _e -> return Nothing
 			Right () -> runner next
 	ReceiveBytes (Len n) next -> do
-		v <- liftIO $ tryIO $ L.hGet (ihdl s) (fromIntegral n)
+		v <- liftIO $ tryIO $ L.hGet (runIhdl runenv) (fromIntegral n)
 		case v of
 			Left _e -> return Nothing
 			Right b -> runner (next b)
-	CheckAuthToken u t next -> do
-		authed <- return True -- TODO XXX FIXME really check
+	CheckAuthToken _u t next -> do
+		let authed = runCheckAuth runenv t
 		runner (next authed)
 	Relay hin hout next -> do
 		v <- liftIO $ runRelay runner hin hout
@@ -88,7 +91,7 @@ runNetHandle s runner f = case f of
 			Nothing -> return Nothing
 			Just exitcode -> runner (next exitcode)
 	RelayService service next -> do
-		v <- liftIO $ runRelayService s runner service
+		v <- liftIO $ runRelayService runenv runner service
 		case v of
 			Nothing -> return Nothing
 			Just () -> runner next
@@ -108,8 +111,8 @@ runRelay runner (RelayHandle hout) (RelayHandle hin) = bracket setup cleanup go
 	
 	go v = relayHelper runner v hin
 
-runRelayService :: S -> RunProto -> Service -> IO (Maybe ())
-runRelayService s runner service = bracket setup cleanup go
+runRelayService :: RunEnv -> RunProto -> Service -> IO (Maybe ())
+runRelayService runenv runner service = bracket setup cleanup go
   where
 	cmd = case service of
 		UploadPack -> "upload-pack"
@@ -117,8 +120,8 @@ runRelayService s runner service = bracket setup cleanup go
 	
 	serviceproc = gitCreateProcess
 		[ Param cmd
-		, File (repoPath (repo s))
-		] (repo s)
+		, File (repoPath (runRepo runenv))
+		] (runRepo runenv)
 
 	setup = do
 		(Just hin, Just hout, _, pid) <- createProcess serviceproc
diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs
index ccb84d1e90..172948d37f 100644
--- a/RemoteDaemon/Transport/Tor.hs
+++ b/RemoteDaemon/Transport/Tor.hs
@@ -12,9 +12,11 @@ import RemoteDaemon.Types
 import RemoteDaemon.Common
 import Utility.Tor
 import Utility.FileMode
+import Utility.AuthToken
 import Remote.Helper.Tor
 import P2P.Protocol
 import P2P.IO
+import P2P.Auth
 import Annex.UUID
 import Types.UUID
 import Messages
@@ -33,7 +35,7 @@ server th@(TransportHandle (LocalRepo r) _) = do
 
 	q <- newTBQueueIO maxConnections
 	replicateM_ maxConnections $
-		forkIO $ forever $ serveClient u r q
+		forkIO $ forever $ serveClient th u r q
 
 	uid <- getRealUserID
 	let ident = fromUUID u
@@ -66,12 +68,21 @@ server th@(TransportHandle (LocalRepo r) _) = do
 maxConnections :: Int
 maxConnections = 10
 
-serveClient :: UUID -> Repo -> TBQueue Handle -> IO ()
-serveClient u r q = bracket setup cleanup go
+serveClient :: TransportHandle -> UUID -> Repo -> TBQueue Handle -> IO ()
+serveClient th u r q = bracket setup cleanup go
   where
 	setup = atomically $ readTBQueue q
 	cleanup = hClose
 	go h = do
 		debugM "remotedaemon" "serving a TOR connection"
-		void $ runNetProtoHandle h h r (serve u)
+		-- Load auth tokens for every connection, to notice
+		-- when the allowed set is changed.
+		allowed <- liftAnnex th loadP2PAuthTokens
+		let runenv = RunEnv
+			{ runRepo = r
+			, runCheckAuth = (`isAllowedAuthToken` allowed)
+			, runIhdl = h
+			, runOhdl = h
+			}
+		void $ runNetProtoHandle runenv (serve u)
 		debugM "remotedaemon" "done with TOR connection"

From 9a0f2d1926f41b0c869ba7f5e63cc16144d5a866 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 30 Nov 2016 17:04:15 -0400
Subject: [PATCH 139/367] devblog

---
 doc/devblog/day_431__p2p_linking.mdwn | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 doc/devblog/day_431__p2p_linking.mdwn

diff --git a/doc/devblog/day_431__p2p_linking.mdwn b/doc/devblog/day_431__p2p_linking.mdwn
new file mode 100644
index 0000000000..1e53ffefc3
--- /dev/null
+++ b/doc/devblog/day_431__p2p_linking.mdwn
@@ -0,0 +1,27 @@
+Today I finished the second-to-last big missing peice for tor hidden service
+remotes. Networks of these remotes are P2P networks, and there needs to be
+a way for peers to find one-another, and to authenticate with one-another.
+The `git annex p2p` command sets up links between peers in such a network.
+
+So far it has only a basic interface that sets up a one way link between
+two peers. In the first repository, run `git annex p2p --gen-address`.
+That outputs a long address. In the second repository, run
+`git annex p2p --link peer1`, and paste the address into it. That sets up a
+git remote named "peer1" that connects back to the first repository over tor.
+
+That is a one-directional link, while a bi-directional link would be
+much more convenient to have between peers. Worse, the address can be reused by
+anyone who sees it, to link into the repository. And, the address is far
+too long to communicate in any way except for pasting it.
+
+So I want to improve that later. What I'd really like to have is an
+interface that displays a one-time-use phrase of five to ten words, that
+can be read over the phone or across the room. Exchange phrases with a
+friend, and get your repositories securely linked together with tor.
+
+But, `git annex p2p` is good enough for now. I can move on to the final
+keystone of the tor support, which is file transfer over tor.
+That should, fingers crossed, be relatively easy, and the `tor` branch is
+close to mergeable now.
+
+Today's work was sponsored by Riku Voipio.

From d0da89d56d6896d193ded6bad47ab448087a6f8c Mon Sep 17 00:00:00 2001
From: spiffytech <spiffytech@web>
Date: Wed, 30 Nov 2016 21:58:01 +0000
Subject: [PATCH 140/367]

---
 doc/forum/Extra___38___missing_folders_on_remote.mdwn | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/forum/Extra___38___missing_folders_on_remote.mdwn

diff --git a/doc/forum/Extra___38___missing_folders_on_remote.mdwn b/doc/forum/Extra___38___missing_folders_on_remote.mdwn
new file mode 100644
index 0000000000..b78961ccc8
--- /dev/null
+++ b/doc/forum/Extra___38___missing_folders_on_remote.mdwn
@@ -0,0 +1,9 @@
+I created a local annex directory that's an adjusted branch used with the assistant. On another machine, I initialized an annex directory, then made this into a full-backup ssh remote for my local.
+
+After the assistant pushes to the remote, and the remote runs `git annex sync`, the remote is missing some directories and has some extra directories. For example, it has the extra directory `Documents/programs/Documents/programs/`, which has different contents than `Documents/programs/`. Both directories are missing the subdirectory `graphing_experiments/`.
+
+From my local, `git annex whereis Documents/programs/graphing_experiments` says the directory exists on the remote. But it's not there.
+
+I recreated the remote from scratch and the problem persists.
+
+The assistant says the remote is caught up, and is keeping up with new content changes. What could cause this?

From 7731f0d9af352bd084bdf1fe7ab57c2147355359 Mon Sep 17 00:00:00 2001
From: "https://anarc.at/openid/" <anarcat@web>
Date: Wed, 30 Nov 2016 22:16:19 +0000
Subject: [PATCH 141/367] Added a comment: magic wormhole

---
 ..._1d5f809564c25e765f82594af8e174ab._comment | 49 +++++++++++++++++++
 1 file changed, 49 insertions(+)
 create mode 100644 doc/devblog/day_431__p2p_linking/comment_1_1d5f809564c25e765f82594af8e174ab._comment

diff --git a/doc/devblog/day_431__p2p_linking/comment_1_1d5f809564c25e765f82594af8e174ab._comment b/doc/devblog/day_431__p2p_linking/comment_1_1d5f809564c25e765f82594af8e174ab._comment
new file mode 100644
index 0000000000..9eceb71edb
--- /dev/null
+++ b/doc/devblog/day_431__p2p_linking/comment_1_1d5f809564c25e765f82594af8e174ab._comment
@@ -0,0 +1,49 @@
+[[!comment format=mdwn
+ username="https://anarc.at/openid/"
+ nickname="anarcat"
+ avatar="http://cdn.libravatar.org/avatar/b36dcf65657dd36128161355d8920a99503def9461c1bb212410980fe6f07125"
+ subject="magic wormhole"
+ date="2016-11-30T22:16:19Z"
+ content="""
+> What I'd really like to have is an interface that displays a
+> one-time-use phrase of five to ten words, that can be read over the
+> phone or across the room. Exchange phrases with a friend, and get
+> your repositories securely linked together with tor.
+
+I already mentionned the project in [[design/assistant/telehash/]],
+but [magic-wormhole](https://github.com/warner/magic-wormhole) does
+exactly that:
+
+    % wormhole send README.md
+    Sending 7924 byte file named 'README.md'
+    On the other computer, please run: wormhole receive
+    Wormhole code is: 7-crossover-clockwork
+    
+    Sending (<-10.0.1.43:58988)..
+    100%|=========================| 7.92K/7.92K [00:00<00:00, 6.02MB/s]
+    File sent.. waiting for confirmation
+    Confirmation received. Transfer complete.
+
+Receiver:
+
+    % wormhole receive
+    Enter receive wormhole code: 7-crossover-clockwork
+    Receiving file (7924 bytes) into: README.md
+    ok? (y/n): y
+    Receiving (->tcp:10.0.1.43:58986)..
+    100%|===========================| 7.92K/7.92K [00:00<00:00, 120KB/s]
+    Received file written to README.md
+
+While that example shows a file transfer, arbitrary data can be
+transfered this way. There's a documented protocol, and it's not
+completely peer-to-peer: there are relay servers to deal with NAT'd
+machines. But the [PAKE
+protocol](https://en.wikipedia.org/wiki/Password-authenticated_key_agreement)
+(basically SPAKE2) could be a good inspiration here.
+
+Otherwise, I must say that, as a user, I don't mind copy-pasting a
+hidden service string (if that's what it's about): i can do that over
+a secure medium (email + OpenPGP or IM + OTR) easily... But I
+understand it can be difficult to do for new users.
+
+"""]]

From 00f48ac4079883a2aeeec4b3a2b9ede101d97c68 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 30 Nov 2016 23:54:00 -0400
Subject: [PATCH 142/367] better comments

---
 P2P/IO.hs | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/P2P/IO.hs b/P2P/IO.hs
index c0b14edca7..82ad23646e 100644
--- a/P2P/IO.hs
+++ b/P2P/IO.hs
@@ -32,6 +32,7 @@ import Control.Concurrent.Async
 import qualified Data.ByteString as B
 import qualified Data.ByteString.Lazy as L
 
+-- Type of interpreters of the Proto free monad.
 type RunProto = forall a m. (MonadIO m, MonadMask m) => Proto a -> m (Maybe a)
 
 data RunEnv = RunEnv
@@ -41,8 +42,10 @@ data RunEnv = RunEnv
 	, runOhdl :: Handle
 	}
 
--- Implementation of the protocol, communicating with a peer
--- over a Handle. No Local actions will be run.
+-- Interpreter of Proto that communicates with a peer over a Handle.
+--
+-- No Local actions will be run; if the interpreter reaches any,
+-- it returns Nothing.
 runNetProtoHandle :: (MonadIO m, MonadMask m) => RunEnv -> Proto a -> m (Maybe a)
 runNetProtoHandle runenv = go
   where
@@ -51,6 +54,9 @@ runNetProtoHandle runenv = go
 	go (Free (Net n)) = runNetHandle runenv go n
 	go (Free (Local _)) = return Nothing
 
+-- Interprater of Net that communicates with a peer over a Handle.
+--
+-- An interpreter for Proto has to be provided.
 runNetHandle :: (MonadIO m, MonadMask m) => RunEnv -> RunProto -> NetF (Proto a) -> m (Maybe a)
 runNetHandle runenv runner f = case f of
 	SendMessage m next -> do

From 94dad1e979f169c772771c37a404d5624a904391 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 1 Dec 2016 00:27:07 -0400
Subject: [PATCH 143/367] more flexible types for Proto runners

This will allow a runner in the Annex monad.
---
 P2P/IO.hs | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/P2P/IO.hs b/P2P/IO.hs
index 82ad23646e..ac7c3e4637 100644
--- a/P2P/IO.hs
+++ b/P2P/IO.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-{-# LANGUAGE RankNTypes, ScopedTypeVariables, CPP #-}
+{-# LANGUAGE RankNTypes, ScopedTypeVariables, FlexibleContexts, CPP #-}
 
 module P2P.IO
 	( RunEnv(..)
@@ -33,7 +33,7 @@ import qualified Data.ByteString as B
 import qualified Data.ByteString.Lazy as L
 
 -- Type of interpreters of the Proto free monad.
-type RunProto = forall a m. (MonadIO m, MonadMask m) => Proto a -> m (Maybe a)
+type RunProto m = forall a. (MonadIO m, MonadMask m) => Proto a -> m (Maybe a)
 
 data RunEnv = RunEnv
 	{ runRepo :: Repo
@@ -49,15 +49,13 @@ data RunEnv = RunEnv
 runNetProtoHandle :: (MonadIO m, MonadMask m) => RunEnv -> Proto a -> m (Maybe a)
 runNetProtoHandle runenv = go
   where
-	go :: RunProto
+	go :: RunProto m
 	go (Pure v) = pure (Just v)
 	go (Free (Net n)) = runNetHandle runenv go n
 	go (Free (Local _)) = return Nothing
 
 -- Interprater of Net that communicates with a peer over a Handle.
---
--- An interpreter for Proto has to be provided.
-runNetHandle :: (MonadIO m, MonadMask m) => RunEnv -> RunProto -> NetF (Proto a) -> m (Maybe a)
+runNetHandle :: (MonadIO m, MonadMask m) => RunEnv -> RunProto m -> NetF (Proto a) -> m (Maybe a)
 runNetHandle runenv runner f = case f of
 	SendMessage m next -> do
 		v <- liftIO $ tryIO $ do
@@ -92,17 +90,23 @@ runNetHandle runenv runner f = case f of
 		let authed = runCheckAuth runenv t
 		runner (next authed)
 	Relay hin hout next -> do
-		v <- liftIO $ runRelay runner hin hout
+		v <- liftIO $ runRelay runnerio hin hout
 		case v of
 			Nothing -> return Nothing
 			Just exitcode -> runner (next exitcode)
 	RelayService service next -> do
-		v <- liftIO $ runRelayService runenv runner service
+		v <- liftIO $ runRelayService runenv runnerio service
 		case v of
 			Nothing -> return Nothing
 			Just () -> runner next
+  where
+	-- This is only used for running Net actions when relaying,
+	-- so it's ok to use runNetProtoHandle, despite it not supporting
+	-- all Proto actions.
+	runnerio :: RunProto IO
+	runnerio = runNetProtoHandle runenv
 
-runRelay :: RunProto -> RelayHandle -> RelayHandle -> IO (Maybe ExitCode)
+runRelay :: RunProto IO -> RelayHandle -> RelayHandle -> IO (Maybe ExitCode)
 runRelay runner (RelayHandle hout) (RelayHandle hin) = bracket setup cleanup go
   where
 	setup = do
@@ -117,7 +121,7 @@ runRelay runner (RelayHandle hout) (RelayHandle hin) = bracket setup cleanup go
 	
 	go v = relayHelper runner v hin
 
-runRelayService :: RunEnv -> RunProto -> Service -> IO (Maybe ())
+runRelayService :: RunEnv -> RunProto IO -> Service -> IO (Maybe ())
 runRelayService runenv runner service = bracket setup cleanup go
   where
 	cmd = case service of
@@ -155,7 +159,7 @@ runRelayService runenv runner service = bracket setup cleanup go
 	waitexit v pid = putMVar v . RelayDone =<< waitForProcess pid
 
 -- Processes RelayData as it is put into the MVar.
-relayHelper :: RunProto -> MVar RelayData -> Handle -> IO (Maybe ExitCode)
+relayHelper :: RunProto IO -> MVar RelayData -> Handle -> IO (Maybe ExitCode)
 relayHelper runner v hin = loop
   where
 	loop = do
@@ -176,7 +180,7 @@ relayHelper runner v hin = loop
 
 -- Takes input from the peer, and puts it into the MVar for processing.
 -- Repeats until the peer tells it it's done or hangs up.
-relayFeeder :: RunProto -> MVar RelayData -> IO ()
+relayFeeder :: RunProto IO -> MVar RelayData -> IO ()
 relayFeeder runner v = loop
   where
 	loop = do

From 3dce6a080e0532e97132eccf81e510d4686dd8a9 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 1 Dec 2016 00:41:01 -0400
Subject: [PATCH 144/367] cleanups

---
 CmdLine/GitRemoteTorAnnex.hs  |  2 +-
 P2P/IO.hs                     | 33 ++++++++++++++++++---------------
 P2P/Protocol.hs               | 32 --------------------------------
 RemoteDaemon/Transport/Tor.hs |  2 +-
 4 files changed, 20 insertions(+), 49 deletions(-)

diff --git a/CmdLine/GitRemoteTorAnnex.hs b/CmdLine/GitRemoteTorAnnex.hs
index 72211c995c..517ce7c82e 100644
--- a/CmdLine/GitRemoteTorAnnex.hs
+++ b/CmdLine/GitRemoteTorAnnex.hs
@@ -66,7 +66,7 @@ connectService address port service = do
 			, runIhdl = h
 			, runOhdl = h
 			}
-		runNetProtoHandle runenv $ do
+		liftIO $ runNetProto runenv $ do
 			v <- auth myuuid authtoken
 			case v of
 				Just _theiruuid -> connect service stdin stdout
diff --git a/P2P/IO.hs b/P2P/IO.hs
index ac7c3e4637..6a834c6597 100644
--- a/P2P/IO.hs
+++ b/P2P/IO.hs
@@ -1,4 +1,4 @@
-{- P2P protocol, partial IO implementation
+{- P2P protocol, IO implementation
  -
  - Copyright 2016 Joey Hess <id@joeyh.name>
  -
@@ -9,8 +9,8 @@
 
 module P2P.IO
 	( RunEnv(..)
-	, runNetProtoHandle
-	, runNetHandle
+	, runNetProto
+	, runNet
 	) where
 
 import P2P.Protocol
@@ -42,21 +42,25 @@ data RunEnv = RunEnv
 	, runOhdl :: Handle
 	}
 
--- Interpreter of Proto that communicates with a peer over a Handle.
+-- Purposefully incomplete interpreter of Proto.
 --
--- No Local actions will be run; if the interpreter reaches any,
+-- This only runs Net actions. No Local actions will be run
+-- (those need the Annex monad) -- if the interpreter reaches any,
 -- it returns Nothing.
-runNetProtoHandle :: (MonadIO m, MonadMask m) => RunEnv -> Proto a -> m (Maybe a)
-runNetProtoHandle runenv = go
+runNetProto :: RunEnv -> Proto a -> IO (Maybe a)
+runNetProto runenv = go
   where
-	go :: RunProto m
+	go :: RunProto IO
 	go (Pure v) = pure (Just v)
-	go (Free (Net n)) = runNetHandle runenv go n
+	go (Free (Net n)) = runNet runenv go n
 	go (Free (Local _)) = return Nothing
 
--- Interprater of Net that communicates with a peer over a Handle.
-runNetHandle :: (MonadIO m, MonadMask m) => RunEnv -> RunProto m -> NetF (Proto a) -> m (Maybe a)
-runNetHandle runenv runner f = case f of
+-- Interprater of the Net part of Proto.
+--
+-- An interpreter of Proto has to be provided, to handle the rest of Proto
+-- actions.
+runNet :: (MonadIO m, MonadMask m) => RunEnv -> RunProto m -> NetF (Proto a) -> m (Maybe a)
+runNet runenv runner f = case f of
 	SendMessage m next -> do
 		v <- liftIO $ tryIO $ do
 			hPutStrLn (runOhdl runenv) (unwords (formatMessage m))
@@ -101,10 +105,9 @@ runNetHandle runenv runner f = case f of
 			Just () -> runner next
   where
 	-- This is only used for running Net actions when relaying,
-	-- so it's ok to use runNetProtoHandle, despite it not supporting
+	-- so it's ok to use runNetProto, despite it not supporting
 	-- all Proto actions.
-	runnerio :: RunProto IO
-	runnerio = runNetProtoHandle runenv
+	runnerio = runNetProto runenv
 
 runRelay :: RunProto IO -> RelayHandle -> RelayHandle -> IO (Maybe ExitCode)
 runRelay runner (RelayHandle hout) (RelayHandle hin) = bracket setup cleanup go
diff --git a/P2P/Protocol.hs b/P2P/Protocol.hs
index 381949af1b..444e08f0e9 100644
--- a/P2P/Protocol.hs
+++ b/P2P/Protocol.hs
@@ -174,38 +174,6 @@ type Local = Free LocalF
 $(makeFree ''NetF)
 $(makeFree ''LocalF)
 
--- | Running Proto actions purely, to see what they do.
-runPure :: Show r => Proto r -> [Message] -> [(String, Maybe Message)]
-runPure (Pure r) _ = [("result: " ++ show r, Nothing)]
-runPure (Free (Net n)) ms = runNet n ms
-runPure (Free (Local n)) ms = runLocal n ms
-
-runNet :: Show r => NetF (Proto r) -> [Message] -> [(String, Maybe Message)]
-runNet (SendMessage m next) ms = (">",  Just m):runPure next ms
-runNet (ReceiveMessage _) [] = [("not enough Messages provided", Nothing)]
-runNet (ReceiveMessage next) (m:ms) = ("<", Just m):runPure (next m) ms
-runNet (SendBytes _ _ next) ms = ("> bytes", Nothing):runPure next ms
-runNet (ReceiveBytes _ next) ms = ("< bytes", Nothing):runPure (next L.empty) ms
-runNet (CheckAuthToken _ _ next) ms = runPure (next True) ms
-runNet (Relay _ _ next) ms = runPure (next ExitSuccess) ms
-runNet (RelayService _ next) ms = runPure next ms
-
-runLocal :: Show r => LocalF (Proto r) -> [Message] -> [(String, Maybe Message)]
-runLocal (KeyFileSize _ next) ms = runPure (next (Len 100)) ms
-runLocal (ReadKeyFile _ _ next) ms = runPure (next L.empty) ms
-runLocal (WriteKeyFile _ _ _ _ next) ms = runPure (next True) ms
-runLocal (SetPresent _ _ next) ms = runPure next ms
-runLocal (CheckContentPresent _ next) ms = runPure (next False) ms
-runLocal (RemoveKeyFile _ next) ms = runPure (next True) ms
-runLocal (TryLockContent _ p next) ms = runPure (p True >> next) ms
-
-protoDump :: [(String, Maybe Message)] -> String
-protoDump = unlines . map protoDump'
-
-protoDump' :: (String, Maybe Message) -> String
-protoDump' (s, Nothing) = s
-protoDump' (s, Just m) = s ++ " " ++ unwords (Proto.formatMessage m)
-
 auth :: UUID -> AuthToken -> Proto (Maybe UUID)
 auth myuuid t = do
 	net $ sendMessage (AUTH myuuid t)
diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs
index 172948d37f..75b1a79230 100644
--- a/RemoteDaemon/Transport/Tor.hs
+++ b/RemoteDaemon/Transport/Tor.hs
@@ -84,5 +84,5 @@ serveClient th u r q = bracket setup cleanup go
 			, runIhdl = h
 			, runOhdl = h
 			}
-		void $ runNetProtoHandle runenv (serve u)
+		void $ runNetProto runenv (serve u)
 		debugM "remotedaemon" "done with TOR connection"

From 4aff2fc6d91c4c2ba888f644758f58f95af05a3e Mon Sep 17 00:00:00 2001
From: "Yury V. Zaytsev" <yury.zaytsev@moneymeets.com>
Date: Tue, 29 Nov 2016 19:49:25 +0100
Subject: [PATCH 145/367] Add initial Jenkins pipeline script

Signed-off-by: Yury V. Zaytsev <yury.zaytsev@moneymeets.com>
---
 Jenkinsfile | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)
 create mode 100644 Jenkinsfile

diff --git a/Jenkinsfile b/Jenkinsfile
new file mode 100644
index 0000000000..66d3753bd7
--- /dev/null
+++ b/Jenkinsfile
@@ -0,0 +1,55 @@
+currentBuild.result = 'SUCCESS'
+def caught_exception = null
+
+final def EMAIL_RECIPIENTS = 'joey+git-annex@joeyh.name'
+
+properties([
+        buildDiscarder(logRotator(artifactNumToKeepStr: '1')),
+        pipelineTriggers([[$class: 'hudson.triggers.SCMTrigger', scmpoll_spec: ''],])  // pollScm('') in 2.22+
+])
+
+try {
+
+    node('windows') {
+
+        dir('git-annex') {
+
+            stage('Checkout') {
+                checkout scm
+            }
+
+            stage('Build') {
+                bat 'c:/msysgit/bin/sh standalone/windows/build.sh'
+            }
+
+            stage('Archive') {
+                archiveArtifacts 'git-annex-installer.exe,dist/build-version'
+            }
+
+            stage('Upload') {
+                withCredentials([usernamePassword(credentialsId: 'rsync-downloads-kitenet-net', passwordVariable: 'RSYNC_PASSWORD', usernameVariable: 'DUMMY')]) {
+                    bat 'c:/cygwin/bin/rsync git-annex-installer.exe winautobuild@downloads.kitenet.net::winautobuild'
+                    bat 'c:/cygwin/bin/rsync dist/build-version winautobuild@downloads.kitenet.net::winautobuild'
+                }
+            }
+
+        }
+
+    }
+
+} catch (exception) {
+
+    caught_exception = exception
+    currentBuild.result = 'FAILURE'
+
+} finally {
+
+    node('master') {
+        step([$class: 'Mailer', notifyEveryUnstableBuild: false, recipients: EMAIL_RECIPIENTS, sendToIndividuals: false])
+    }
+
+    if (caught_exception) {
+        throw caught_exception
+    }
+
+}

From ddaea54191d3b07b2a35ac6f0b63d93a3c58f73c Mon Sep 17 00:00:00 2001
From: "Yury V. Zaytsev" <yury.zaytsev@moneymeets.com>
Date: Wed, 30 Nov 2016 19:37:32 +0100
Subject: [PATCH 146/367] Remove obsolete link to Windows build logs

---
 doc/install/Windows.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/install/Windows.mdwn b/doc/install/Windows.mdwn
index c9b60d6428..1d9599ac84 100644
--- a/doc/install/Windows.mdwn
+++ b/doc/install/Windows.mdwn
@@ -24,7 +24,7 @@ important thing is that it should end with "All tests passed".
 A daily build is also available, thanks to Yury V. Zaytsev and
 Dartmouth College.
 
-* [download](https://downloads.kitenet.net/git-annex/autobuild/windows/) ([build logs](https://qa.nest-initiative.org/view/msysGit/job/msysgit-git-annex-assistant-test/))
+* [download](https://downloads.kitenet.net/git-annex/autobuild/windows/)
 
 ## building it yourself
 

From 020d5ecbfcd3cdf44b1892b993c9759df3cbbf25 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 1 Dec 2016 12:07:21 -0400
Subject: [PATCH 147/367] change link to windows autobuild page

---
 doc/builds.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/builds.mdwn b/doc/builds.mdwn
index e6c7c80825..77c9351e3a 100644
--- a/doc/builds.mdwn
+++ b/doc/builds.mdwn
@@ -49,7 +49,7 @@
 <iframe width=1024 scrolling=no frameborder=0 marginheight=0 marginwidth=0 src="https://downloads.kitenet.net/git-annex/autobuildtest/x86_64-apple-yosemite/testresult/status">
 </iframe>
 <h2><a href="https://qa.nest-initiative.org/view/msysGit/job/msysgit-git-annex-assistant-test/">Windows</a></h2>
-<a href="https://qa.nest-initiative.org/view/msysGit/job/msysgit-git-annex-assistant-test/">here</a>
+<a href="http://vps.zaytsev.net:8080/">here</a> (firewalled from most locations; kite can access it)
 <h2><a href="https://buildd.debian.org/status/package.php?p=git-annex&suite=sid">Debian</a></h2>
 <iframe width=1024 scrolling=no height=500px frameborder=0 marginheight=0 marginwidth=0 src="https://buildd.debian.org/status/package.php?p=git-annex&suite=sid">
 </iframe>

From be5cd15311d8c43834299ca63caa242783c50348 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 1 Dec 2016 12:17:31 -0400
Subject: [PATCH 148/367] urk

test failures were being ignored for complex reasons, new autobuilder broke that; bring it back
---
 standalone/windows/build.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/standalone/windows/build.sh b/standalone/windows/build.sh
index 217a0bf827..a8469a675b 100755
--- a/standalone/windows/build.sh
+++ b/standalone/windows/build.sh
@@ -105,4 +105,7 @@ export PATH
 mkdir -p c:/WINDOWS/Temp/git-annex-test/
 cd c:/WINDOWS/Temp/git-annex-test/
 rm -rf .t
-withcyg git-annex.exe test
+# Currently the test fails in the autobuilder environment for reasons not
+# yet understood. Windows users are encouraged to run the test suite
+# themseves, so we'll ignore these failures for now.
+withcyg git-annex.exe test || true

From 1f26a94cb98ea702445c5e0b4093ce706e315d67 Mon Sep 17 00:00:00 2001
From: 
 "https://openid.stackexchange.com/user/3ee5cf54-f022-4a71-8666-3c2b5ee231dd"
 <Anthony_DeRobertis@web>
Date: Thu, 1 Dec 2016 18:54:24 +0000
Subject: [PATCH 149/367]

---
 doc/bugs/ssh___34__Include__34___feature_broke_Android.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 doc/bugs/ssh___34__Include__34___feature_broke_Android.mdwn

diff --git a/doc/bugs/ssh___34__Include__34___feature_broke_Android.mdwn b/doc/bugs/ssh___34__Include__34___feature_broke_Android.mdwn
new file mode 100644
index 0000000000..8cdf8e1577
--- /dev/null
+++ b/doc/bugs/ssh___34__Include__34___feature_broke_Android.mdwn
@@ -0,0 +1,6 @@
+### Please describe the problem.
+https://git-annex.branchable.com/bugs/git_annex_init_failed_due_to_unsupported_ssh_option/ deal with Include not being supported by pre 7.3 by using the 6.4+ IgnoreUnknown directive.
+
+Unfortunately, the Android apk (which I got from https://downloads.kitenet.net/git-annex/android/current/5.0/git-annex.apk) has (according to ssh -v) OpenSSH_6.0p1.
+
+I had to edit .git/annex/ssh.config to comment out the three offending lines and then append the contents of ~/.ssh/config to get git-annex working again.

From 1a1bbeb659c1d7ec24d4684c729746350cc2bbd3 Mon Sep 17 00:00:00 2001
From: 
 "https://openid.stackexchange.com/user/3ee5cf54-f022-4a71-8666-3c2b5ee231dd"
 <Anthony_DeRobertis@web>
Date: Thu, 1 Dec 2016 18:54:57 +0000
Subject: [PATCH 150/367]

---
 doc/bugs/ssh___34__Include__34___feature_broke_Android.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/ssh___34__Include__34___feature_broke_Android.mdwn b/doc/bugs/ssh___34__Include__34___feature_broke_Android.mdwn
index 8cdf8e1577..9be4d4318d 100644
--- a/doc/bugs/ssh___34__Include__34___feature_broke_Android.mdwn
+++ b/doc/bugs/ssh___34__Include__34___feature_broke_Android.mdwn
@@ -4,3 +4,5 @@ https://git-annex.branchable.com/bugs/git_annex_init_failed_due_to_unsupported_s
 Unfortunately, the Android apk (which I got from https://downloads.kitenet.net/git-annex/android/current/5.0/git-annex.apk) has (according to ssh -v) OpenSSH_6.0p1.
 
 I had to edit .git/annex/ssh.config to comment out the three offending lines and then append the contents of ~/.ssh/config to get git-annex working again.
+
+(This is on a Nexus 10 running stock, though I doubt it matters)

From 15dc63d47f59b2e0e153b3d19da1300614f6c8c5 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 2 Dec 2016 13:45:45 -0400
Subject: [PATCH 151/367] make sure that the specified number of bytes of DATA
 are always sent

It's possible, in direct or thin mode, that an object file gets
truncated or appended to as it's being sent. This would break the
protocol badly, so make sure never to send too many bytes, and to
close the protocol connection if too few bytes are available.
---
 P2P/IO.hs | 40 +++++++++++++++++++++++++++++++++-------
 1 file changed, 33 insertions(+), 7 deletions(-)

diff --git a/P2P/IO.hs b/P2P/IO.hs
index 6a834c6597..6ab6cc2787 100644
--- a/P2P/IO.hs
+++ b/P2P/IO.hs
@@ -5,10 +5,11 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-{-# LANGUAGE RankNTypes, ScopedTypeVariables, FlexibleContexts, CPP #-}
+{-# LANGUAGE RankNTypes, FlexibleContexts, BangPatterns, CPP #-}
 
 module P2P.IO
-	( RunEnv(..)
+	( RunProto
+	, RunEnv(..)
 	, runNetProto
 	, runNet
 	) where
@@ -55,7 +56,7 @@ runNetProto runenv = go
 	go (Free (Net n)) = runNet runenv go n
 	go (Free (Local _)) = return Nothing
 
--- Interprater of the Net part of Proto.
+-- Interpreter of the Net part of Proto.
 --
 -- An interpreter of Proto has to be provided, to handle the rest of Proto
 -- actions.
@@ -78,13 +79,14 @@ runNet runenv runner f = case f of
 					let e = ERROR $ "protocol parse error: " ++ show l
 					net $ sendMessage e
 					next e
-	SendBytes _len b next -> do
+	SendBytes len b next -> do
 		v <- liftIO $ tryIO $ do
-			L.hPut (runOhdl runenv) b
+			ok <- sendExactly len b (runOhdl runenv)
 			hFlush (runOhdl runenv)
+			return ok
 		case v of
-			Left _e -> return Nothing
-			Right () -> runner next
+			Right True -> runner next
+			_ -> return Nothing
 	ReceiveBytes (Len n) next -> do
 		v <- liftIO $ tryIO $ L.hGet (runIhdl runenv) (fromIntegral n)
 		case v of
@@ -109,6 +111,30 @@ runNet runenv runner f = case f of
 	-- all Proto actions.
 	runnerio = runNetProto runenv
 
+-- Send exactly the specified number of bytes or returns False.
+--
+-- The ByteString can be larger or smaller than the specified length.
+-- For example, it can be lazily streaming from a file that gets
+-- appended to, or truncated.
+--
+-- Must avoid sending too many bytes as it would confuse the other end.
+-- This is easily dealt with by truncating it.
+--
+-- If too few bytes are sent, the only option is to give up on this
+-- connection. False is returned to indicate this problem.
+--
+-- We can't check the length of the whole lazy bytestring without buffering
+-- it in memory. Instead, process it one chunk at a time, and sum the length
+-- of the chunks.
+sendExactly :: Len -> L.ByteString -> Handle -> IO Bool
+sendExactly (Len l) lb h = go 0 $ L.toChunks $ L.take (fromIntegral l) lb
+  where
+	go n [] = return (toInteger n == l)
+	go n (b:bs) = do
+		B.hPut h b
+		let !n' = n + B.length b
+		go n' bs
+
 runRelay :: RunProto IO -> RelayHandle -> RelayHandle -> IO (Maybe ExitCode)
 runRelay runner (RelayHandle hout) (RelayHandle hin) = bracket setup cleanup go
   where

From 7b7afbbedce67ee40c7c5c8359fd71991c1aed0f Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 2 Dec 2016 13:47:42 -0400
Subject: [PATCH 152/367] improve Local monad

---
 P2P/Protocol.hs | 65 +++++++++++++++++++++++++++++--------------------
 1 file changed, 38 insertions(+), 27 deletions(-)

diff --git a/P2P/Protocol.hs b/P2P/Protocol.hs
index 444e08f0e9..7c83a26b17 100644
--- a/P2P/Protocol.hs
+++ b/P2P/Protocol.hs
@@ -127,7 +127,12 @@ data NetF c
 	= SendMessage Message c
 	| ReceiveMessage (Message -> c)
 	| SendBytes Len L.ByteString c
+	-- ^ Sends exactly Len bytes of data. (Any more or less will
+	-- confuse the receiver.)
 	| ReceiveBytes Len (L.ByteString -> c)
+	-- ^ Lazily reads bytes from peer. Stops once Len are read,
+	-- or if connection is lost, and in either case returns the bytes
+	-- that were read. This allows resuming interrupted transfers.
 	| CheckAuthToken UUID AuthToken (Bool -> c)
 	| RelayService Service c
 	-- ^ Runs a service, relays its output to the peer, and data
@@ -144,24 +149,28 @@ type Net = Free NetF
 newtype RelayHandle = RelayHandle Handle
 
 data LocalF c
-	-- ^ Lazily reads bytes from peer. Stops once Len are read,
-	-- or if connection is lost, and in either case returns the bytes
-	-- that were read. This allows resuming interrupted transfers.
-	= KeyFileSize Key (Len -> c)
-	-- ^ Checks size of key file (dne = 0)
-	| ReadKeyFile Key Offset (L.ByteString -> c)
-	| WriteKeyFile Key Offset Len L.ByteString (Bool -> c)
-	-- ^ Writes to key file starting at an offset. Returns True
-	-- once the whole content of the key is stored in the key file.
+	= TmpContentSize Key (Len -> c)
+	-- ^ Gets size of the temp file where received content may have
+	-- been stored. If not present, returns 0.
+	| ContentSize Key (Maybe Len -> c)
+	-- ^ Gets size of the content of a key, when the full content is
+	-- present.
+	| ReadContent Key Offset (L.ByteString -> c)
+	-- ^ Lazily reads the content of a key. Note that the content
+	-- may change while it's being sent.
+	| WriteContent Key Offset Len L.ByteString (Bool -> c)
+	-- ^ Writes content to temp file starting at an offset.
+	-- Once the whole content of the key has been stored, moves the
+	-- temp file into place and returns True.
 	--
 	-- Note: The ByteString may not contain the entire remaining content
-	-- of the key. Only once the key file size == Len has the whole
+	-- of the key. Only once the temp file size == Len has the whole
 	-- content been transferred.
 	| SetPresent Key UUID c
 	| CheckContentPresent Key (Bool -> c)
 	-- ^ Checks if the whole content of the key is locally present.
-	| RemoveKeyFile Key (Bool -> c)
-	-- ^ If the key file is not present, still succeeds.
+	| RemoveContent Key (Bool -> c)
+	-- ^ If the content is not present, still succeeds.
 	-- May fail if not enough copies to safely drop, etc.
 	| TryLockContent Key (Bool -> Proto ()) c
 	-- ^ Try to lock the content of a key,  preventing it
@@ -272,7 +281,7 @@ serve myuuid = go Nothing
 					UNLOCKCONTENT -> return ()
 					_ -> net $ sendMessage (ERROR "expected UNLOCKCONTENT")
 		CHECKPRESENT key -> sendSuccess =<< local (checkContentPresent key)
-		REMOVE key ->  sendSuccess =<< local (removeKeyFile key)
+		REMOVE key ->  sendSuccess =<< local (removeContent key)
 		PUT key -> do
 			have <- local $ checkContentPresent key
 			if have
@@ -289,20 +298,20 @@ serve myuuid = go Nothing
 
 sendContent :: Key -> Offset -> Proto Bool
 sendContent key offset = do
-	(len, content) <- readKeyFileLen key offset
+	(len, content) <- readContentLen key offset
 	net $ sendMessage (DATA len)
 	net $ sendBytes len content
 	checkSuccess
 
 receiveContent :: Key -> (Offset -> Message) -> Proto Bool
 receiveContent key mkmsg = do
-	Len n <- local $ keyFileSize key
+	Len n <- local $ tmpContentSize key
 	let offset = Offset n
 	net $ sendMessage (mkmsg offset)
 	r <- net receiveMessage
 	case r of
 		DATA len -> do
-			ok <- local . writeKeyFile key offset len
+			ok <- local . writeContent key offset len
 				=<< net (receiveBytes len)
 			sendSuccess ok
 			return ok
@@ -324,18 +333,20 @@ sendSuccess :: Bool -> Proto ()
 sendSuccess True = net $ sendMessage SUCCESS
 sendSuccess False = net $ sendMessage FAILURE
 
--- Reads key file from an offset. The Len should correspond to
+-- Reads content from an offset. The Len should correspond to
 -- the length of the ByteString, but to avoid buffering the content
--- in memory, is gotten using keyFileSize.
-readKeyFileLen :: Key -> Offset -> Proto (Len, L.ByteString)
-readKeyFileLen key (Offset offset) = do
-	(Len totallen) <- local $ keyFileSize key
-	let len = totallen - offset
-	if len <= 0
-		then return (Len 0, L.empty)
-		else do
-			content <- local $ readKeyFile key (Offset offset)
-			return (Len len, content)
+-- in memory, is gotten using contentSize.
+readContentLen :: Key -> Offset -> Proto (Len, L.ByteString)
+readContentLen key (Offset offset) = go =<< local (contentSize key)
+  where
+	go Nothing = return (Len 0, L.empty)
+	go (Just (Len totallen)) = do
+		let len = totallen - offset
+		if len <= 0
+			then return (Len 0, L.empty)
+			else do
+				content <- local $ readContent key (Offset offset)
+				return (Len len, content)
 
 connect :: Service -> Handle -> Handle -> Proto ExitCode
 connect service hin hout = do

From 881274d02164232fb363ae706b73ace2ef21a5b6 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 2 Dec 2016 13:50:56 -0400
Subject: [PATCH 153/367] make remote-daemon able to send and receive objects
 over tor

Each worker thread needs to run in the Annex monad, but the
remote-daemon's liftAnnex can only run 1 action at a time. Used
Annex.Concurrent to deal with that.

P2P.Annex is incomplete as of yet.
---
 P2P/Annex.hs                  | 36 +++++++++++++++++++++++++++++++++++
 RemoteDaemon/Transport/Tor.hs | 30 ++++++++++++++++++-----------
 git-annex.cabal               |  1 +
 3 files changed, 56 insertions(+), 11 deletions(-)
 create mode 100644 P2P/Annex.hs

diff --git a/P2P/Annex.hs b/P2P/Annex.hs
new file mode 100644
index 0000000000..ad4b458dd8
--- /dev/null
+++ b/P2P/Annex.hs
@@ -0,0 +1,36 @@
+{- P2P protocol, Annex implementation
+ -
+ - Copyright 2016 Joey Hess <id@joeyh.name>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE RankNTypes, FlexibleContexts #-}
+
+module P2P.Annex
+	( RunEnv(..)
+	, runFullProto
+	) where
+
+import Annex.Common
+import Annex.Content
+import P2P.Protocol
+import P2P.IO
+
+import Control.Monad.Free
+
+-- Full interpreter for Proto, that can receive and send objects.
+runFullProto :: RunEnv -> Proto a -> Annex (Maybe a)
+runFullProto runenv = go
+  where
+	go :: RunProto Annex
+	go (Pure v) = pure (Just v)
+	go (Free (Net n)) = runNet runenv go n
+	go (Free (Local l)) = runLocal runenv go l
+
+runLocal :: RunEnv -> RunProto Annex -> LocalF (Proto a) -> Annex (Maybe a)
+runLocal runenv runner f = case f of
+	TmpContentSize k next -> do
+		tmp <- fromRepo $ gitAnnexTmpObjectLocation k
+		size <- liftIO $ catchDefaultIO 0 $ getFileSize tmp
+		runner (next (Len size))
diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs
index 75b1a79230..3c715fbde1 100644
--- a/RemoteDaemon/Transport/Tor.hs
+++ b/RemoteDaemon/Transport/Tor.hs
@@ -8,6 +8,8 @@
 module RemoteDaemon.Transport.Tor (server) where
 
 import Common
+import qualified Annex
+import Annex.Concurrent
 import RemoteDaemon.Types
 import RemoteDaemon.Common
 import Utility.Tor
@@ -15,7 +17,7 @@ import Utility.FileMode
 import Utility.AuthToken
 import Remote.Helper.Tor
 import P2P.Protocol
-import P2P.IO
+import P2P.Annex
 import P2P.Auth
 import Annex.UUID
 import Types.UUID
@@ -75,14 +77,20 @@ serveClient th u r q = bracket setup cleanup go
 	cleanup = hClose
 	go h = do
 		debugM "remotedaemon" "serving a TOR connection"
-		-- Load auth tokens for every connection, to notice
-		-- when the allowed set is changed.
-		allowed <- liftAnnex th loadP2PAuthTokens
-		let runenv = RunEnv
-			{ runRepo = r
-			, runCheckAuth = (`isAllowedAuthToken` allowed)
-			, runIhdl = h
-			, runOhdl = h
-			}
-		void $ runNetProto runenv (serve u)
+		-- Avoid doing any work in the liftAnnex, since only one
+		-- can run at a time.
+		st <- liftAnnex th dupState
+		((), st') <- Annex.run st $ do
+			-- Load auth tokens for every connection, to notice
+			-- when the allowed set is changed.
+			allowed <- loadP2PAuthTokens
+			let runenv = RunEnv
+				{ runRepo = r
+				, runCheckAuth = (`isAllowedAuthToken` allowed)
+				, runIhdl = h
+				, runOhdl = h
+				}
+			void $ runFullProto runenv (serve u)
+		-- Merge the duplicated state back in.
+		liftAnnex th $ mergeState st'
 		debugM "remotedaemon" "done with TOR connection"
diff --git a/git-annex.cabal b/git-annex.cabal
index 6991d2a048..f6d8c54823 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -908,6 +908,7 @@ Executable git-annex
     Messages.JSON
     Messages.Progress
     P2P.Address
+    P2P.Annex
     P2P.Auth
     P2P.IO
     P2P.Protocol

From c29f2e262aaaef27182a4490b1c0dc5e56758b4f Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 2 Dec 2016 14:16:50 -0400
Subject: [PATCH 154/367] catch non-IO exceptions too

---
 P2P/IO.hs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/P2P/IO.hs b/P2P/IO.hs
index 6ab6cc2787..fb621ab2b1 100644
--- a/P2P/IO.hs
+++ b/P2P/IO.hs
@@ -63,14 +63,14 @@ runNetProto runenv = go
 runNet :: (MonadIO m, MonadMask m) => RunEnv -> RunProto m -> NetF (Proto a) -> m (Maybe a)
 runNet runenv runner f = case f of
 	SendMessage m next -> do
-		v <- liftIO $ tryIO $ do
+		v <- liftIO $ tryNonAsync $ do
 			hPutStrLn (runOhdl runenv) (unwords (formatMessage m))
 			hFlush (runOhdl runenv)
 		case v of
 			Left _e -> return Nothing
 			Right () -> runner next
 	ReceiveMessage next -> do
-		v <- liftIO $ tryIO $ hGetLine (runIhdl runenv)
+		v <- liftIO $ tryNonAsync $ hGetLine (runIhdl runenv)
 		case v of
 			Left _e -> return Nothing
 			Right l -> case parseMessage l of
@@ -80,7 +80,7 @@ runNet runenv runner f = case f of
 					net $ sendMessage e
 					next e
 	SendBytes len b next -> do
-		v <- liftIO $ tryIO $ do
+		v <- liftIO $ tryNonAsync $ do
 			ok <- sendExactly len b (runOhdl runenv)
 			hFlush (runOhdl runenv)
 			return ok
@@ -88,7 +88,7 @@ runNet runenv runner f = case f of
 			Right True -> runner next
 			_ -> return Nothing
 	ReceiveBytes (Len n) next -> do
-		v <- liftIO $ tryIO $ L.hGet (runIhdl runenv) (fromIntegral n)
+		v <- liftIO $ tryNonAsync $ L.hGet (runIhdl runenv) (fromIntegral n)
 		case v of
 			Left _e -> return Nothing
 			Right b -> runner (next b)

From 71ddb10699663f2a9e80a796fd5e96a612584672 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 2 Dec 2016 14:49:22 -0400
Subject: [PATCH 155/367] initial implementation of P2P.Annex runner

Untested, and it does not yet update transfer logs.

Verifying transferred content is modeled on git-annex-shell recvkey.
In a direct mode or annex.thin repository, content can change while it's
being transferred. So, verification is always done, even if annex.verify
would normally prevent it.

Note that a WORM or URL key could change in a way the verification
doesn't catch. That can happen in git-annex-shell recvkey too. We don't
worry about it, because those key backends don't guarantee preservation
of data. (Which is to say, I worried about it, and then convinced myself
again it was ok.)
---
 P2P/Annex.hs    | 72 ++++++++++++++++++++++++++++++++++++++++++++++---
 P2P/Protocol.hs |  6 +++--
 2 files changed, 73 insertions(+), 5 deletions(-)

diff --git a/P2P/Annex.hs b/P2P/Annex.hs
index ad4b458dd8..d9ea530f05 100644
--- a/P2P/Annex.hs
+++ b/P2P/Annex.hs
@@ -16,8 +16,11 @@ import Annex.Common
 import Annex.Content
 import P2P.Protocol
 import P2P.IO
+import Logs.Location
+import Types.NumCopies
 
 import Control.Monad.Free
+import qualified Data.ByteString.Lazy as L
 
 -- Full interpreter for Proto, that can receive and send objects.
 runFullProto :: RunEnv -> Proto a -> Annex (Maybe a)
@@ -26,11 +29,74 @@ runFullProto runenv = go
 	go :: RunProto Annex
 	go (Pure v) = pure (Just v)
 	go (Free (Net n)) = runNet runenv go n
-	go (Free (Local l)) = runLocal runenv go l
+	go (Free (Local l)) = runLocal go l
 
-runLocal :: RunEnv -> RunProto Annex -> LocalF (Proto a) -> Annex (Maybe a)
-runLocal runenv runner f = case f of
+runLocal :: RunProto Annex -> LocalF (Proto a) -> Annex (Maybe a)
+runLocal runner a = case a of
 	TmpContentSize k next -> do
 		tmp <- fromRepo $ gitAnnexTmpObjectLocation k
 		size <- liftIO $ catchDefaultIO 0 $ getFileSize tmp
 		runner (next (Len size))
+	ContentSize k next -> do
+		let getsize = liftIO . catchMaybeIO . getFileSize
+		size <- inAnnex' isJust Nothing getsize k
+		runner (next (Len <$> size))
+	-- TODO transfer logs
+	ReadContent k (Offset o) next -> do
+		v <- tryNonAsync $ prepSendAnnex k
+		case v of
+			-- The check can detect a problem after the
+			-- content is sent, but we don't use it.
+			-- Instead, the receiving peer must AlwaysVerify
+			-- the content it receives.
+			Right (Just (f, _check)) -> do
+				v' <- liftIO $ tryNonAsync $ do
+					h <- openBinaryFile f ReadMode
+					when (o /= 0) $
+						hSeek h AbsoluteSeek o
+					L.hGetContents h
+				case v' of
+					Left _ -> return Nothing
+					Right b -> runner (next b)
+			_ -> return Nothing
+	-- TODO transfer logs
+	WriteContent k (Offset o) (Len l) b next -> do
+		ok <- flip catchNonAsync (const $ return False) $
+			getViaTmp AlwaysVerify k $ \tmp -> liftIO $ do
+				withBinaryFile tmp WriteMode $ \h -> do
+					when (o /= 0) $
+						hSeek h AbsoluteSeek o
+					L.hPut h b
+				sz <- getFileSize tmp
+				return (toInteger sz == l, UnVerified)
+		runner (next ok)
+	SetPresent k u next -> do
+		v <- tryNonAsync $ logChange k u InfoPresent
+		case v of
+			Left _ -> return Nothing
+			Right () -> runner next
+	CheckContentPresent k next -> do
+		v <- tryNonAsync $ inAnnex k
+		case v of
+			Left _ -> return Nothing
+			Right result -> runner (next result)
+	RemoveContent k next -> do
+		v <- tryNonAsync $ lockContentForRemoval k $ \contentlock -> do
+				removeAnnex contentlock
+				logStatus k InfoMissing
+				return True
+		case v of
+			Left _ -> return Nothing
+			Right result -> runner (next result)
+	TryLockContent k protoaction next -> do
+		v <- tryNonAsync $ lockContentShared k $ \verifiedcopy -> 
+			case verifiedcopy of
+				LockedCopy _ -> runner (protoaction True)
+				_ -> runner (protoaction False)
+		-- If locking fails, lockContentShared throws an exception.
+		-- Let the peer know it failed.
+		case v of
+			Left _ -> runner $ do
+				protoaction False
+				next
+			Right _ -> runner next
diff --git a/P2P/Protocol.hs b/P2P/Protocol.hs
index 7c83a26b17..b67dc118d3 100644
--- a/P2P/Protocol.hs
+++ b/P2P/Protocol.hs
@@ -174,7 +174,9 @@ data LocalF c
 	-- May fail if not enough copies to safely drop, etc.
 	| TryLockContent Key (Bool -> Proto ()) c
 	-- ^ Try to lock the content of a key,  preventing it
-	-- from being deleted, and run the provided protocol action.
+	-- from being deleted, while running the provided protocol
+	-- action. If unable to lock the content, runs the protocol action
+	-- with False.
 	deriving (Functor)
 
 type Local = Free LocalF
@@ -291,7 +293,7 @@ serve myuuid = go Nothing
 					when ok $
 						local $ setPresent key myuuid
 		-- setPresent not called because the peer may have
-		-- requested the data but not permanatly stored it.
+		-- requested the data but not permanently stored it.
 		GET offset key -> void $ sendContent key offset
 		CONNECT service -> net $ relayService service
 		_ -> net $ sendMessage (ERROR "unexpected command")

From b16a1cee4be6d44a644b2a788a73cc75d40afe4c Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 2 Dec 2016 15:34:15 -0400
Subject: [PATCH 156/367] plumb peer uuid through to runLocal

This will allow updating transfer logs with the uuid.
---
 P2P/Annex.hs                  |  19 ++++--
 P2P/Protocol.hs               | 124 +++++++++++++++++++++-------------
 RemoteDaemon/Transport/Tor.hs |   8 ++-
 3 files changed, 96 insertions(+), 55 deletions(-)

diff --git a/P2P/Annex.hs b/P2P/Annex.hs
index d9ea530f05..dce4ceebac 100644
--- a/P2P/Annex.hs
+++ b/P2P/Annex.hs
@@ -8,7 +8,8 @@
 {-# LANGUAGE RankNTypes, FlexibleContexts #-}
 
 module P2P.Annex
-	( RunEnv(..)
+	( RunMode(..)
+	, RunEnv(..)
 	, runFullProto
 	) where
 
@@ -22,17 +23,23 @@ import Types.NumCopies
 import Control.Monad.Free
 import qualified Data.ByteString.Lazy as L
 
+-- When we're serving a peer, we know their uuid, and can use it to update
+-- transfer logs.
+data RunMode
+	= Serving UUID
+	| Client
+
 -- Full interpreter for Proto, that can receive and send objects.
-runFullProto :: RunEnv -> Proto a -> Annex (Maybe a)
-runFullProto runenv = go
+runFullProto :: RunMode -> RunEnv -> Proto a -> Annex (Maybe a)
+runFullProto runmode runenv = go
   where
 	go :: RunProto Annex
 	go (Pure v) = pure (Just v)
 	go (Free (Net n)) = runNet runenv go n
-	go (Free (Local l)) = runLocal go l
+	go (Free (Local l)) = runLocal runmode go l
 
-runLocal :: RunProto Annex -> LocalF (Proto a) -> Annex (Maybe a)
-runLocal runner a = case a of
+runLocal :: RunMode -> RunProto Annex -> LocalF (Proto a) -> Annex (Maybe a)
+runLocal runmode runner a = case a of
 	TmpContentSize k next -> do
 		tmp <- fromRepo $ gitAnnexTmpObjectLocation k
 		size <- liftIO $ catchDefaultIO 0 $ getFileSize tmp
diff --git a/P2P/Protocol.hs b/P2P/Protocol.hs
index b67dc118d3..9678b7954b 100644
--- a/P2P/Protocol.hs
+++ b/P2P/Protocol.hs
@@ -240,63 +240,91 @@ put key = do
 			net $ sendMessage (ERROR "expected PUT_FROM")
 			return False
 
--- | Serve the protocol.
---
--- Note that if the client sends an unexpected message, the server will
--- respond with PTOTO_ERROR, and always continues processing messages.
--- Since the protocol is not versioned, this is necessary to handle
--- protocol changes robustly, since the client can detect when it's
--- talking to a server that does not support some new feature, and fall
--- back.
---
--- When the client sends ERROR to the server, the server gives up,
--- since it's not clear what state the client is is, and so not possible to
--- recover.
-serve :: UUID -> Proto ()
-serve myuuid = go Nothing
+data ServerHandler a
+	= ServerGot a
+	| ServerContinue
+	| ServerUnexpected
+
+-- Server loop, getting messages from the client and handling them
+serverLoop :: (Message -> Proto (ServerHandler a)) -> Proto (Maybe a)
+serverLoop a = do
+	cmd <- net receiveMessage
+	case cmd of
+		-- When the client sends ERROR to the server, the server
+		-- gives up, since it's not clear what state the client
+		-- is in, and so not possible to recover.
+		ERROR _ -> return Nothing
+		_ -> do
+			v <- a cmd
+			case v of
+				ServerGot r -> return (Just r)
+				ServerContinue -> serverLoop a
+				-- If the client sends an unexpected message,
+				-- the server will respond with ERROR, and
+				-- always continues processing messages.
+				--
+				-- Since the protocol is not versioned, this
+				-- is necessary to handle protocol changes
+				-- robustly, since the client can detect when
+				-- it's talking to a server that does not
+				-- support some new feature, and fall back.
+				ServerUnexpected -> do
+					net $ sendMessage (ERROR "unexpected command")
+					serverLoop a
+
+-- | Serve the protocol, with an unauthenticated peer. Once the peer
+-- successfully authenticates, returns their UUID.
+serveAuth :: UUID -> Proto (Maybe UUID)
+serveAuth myuuid = serverLoop handler
   where
-	go autheduuid = do
-		r <- net receiveMessage
-		case r of
-			AUTH theiruuid authtoken -> do
-				ok <- net $ checkAuthToken theiruuid authtoken
-				if ok
-					then do
-						net $ sendMessage (AUTH_SUCCESS myuuid)
-						go (Just theiruuid)
-					else do
-						net $ sendMessage AUTH_FAILURE
-						go autheduuid
-			ERROR _ -> return ()
-			_ -> do
-				case autheduuid of
-					Just theiruuid -> authed theiruuid r
-					Nothing -> net $ sendMessage (ERROR "must AUTH first")
-				go autheduuid
-	
-	authed _theiruuid r = case r of
-		LOCKCONTENT key -> local $ tryLockContent key $ \locked -> do
+	handler (AUTH theiruuid authtoken) = do
+		ok <- net $ checkAuthToken theiruuid authtoken
+		if ok
+			then do
+				net $ sendMessage (AUTH_SUCCESS myuuid)
+				return (ServerGot theiruuid)
+			else do
+				net $ sendMessage AUTH_FAILURE
+				return ServerContinue
+	handler _ = return ServerUnexpected
+
+-- | Serve the protocol, with a peer that has authenticated.
+serveAuthed :: UUID -> Proto ()
+serveAuthed myuuid = void $ serverLoop handler
+  where
+	handler (LOCKCONTENT key) = do
+		local $ tryLockContent key $ \locked -> do
 			sendSuccess locked
 			when locked $ do
 				r' <- net receiveMessage
 				case r' of
 					UNLOCKCONTENT -> return ()
 					_ -> net $ sendMessage (ERROR "expected UNLOCKCONTENT")
-		CHECKPRESENT key -> sendSuccess =<< local (checkContentPresent key)
-		REMOVE key ->  sendSuccess =<< local (removeContent key)
-		PUT key -> do
-			have <- local $ checkContentPresent key
-			if have
-				then net $ sendMessage ALREADY_HAVE
-				else do
-					ok <- receiveContent key PUT_FROM
-					when ok $
-						local $ setPresent key myuuid
+		return ServerContinue
+	handler (CHECKPRESENT key) = do
+		sendSuccess =<< local (checkContentPresent key)
+		return ServerContinue
+	handler (REMOVE key) = do
+		sendSuccess =<< local (removeContent key)
+		return ServerContinue
+	handler (PUT key) = do
+		have <- local $ checkContentPresent key
+		if have
+			then net $ sendMessage ALREADY_HAVE
+			else do
+				ok <- receiveContent key PUT_FROM
+				when ok $
+					local $ setPresent key myuuid
+		return ServerContinue
+	handler (GET offset key) = do
+		void $ sendContent key offset
 		-- setPresent not called because the peer may have
 		-- requested the data but not permanently stored it.
-		GET offset key -> void $ sendContent key offset
-		CONNECT service -> net $ relayService service
-		_ -> net $ sendMessage (ERROR "unexpected command")
+		return ServerContinue
+	handler (CONNECT service) = do
+		net $ relayService service
+		return ServerContinue
+	handler _ = return ServerUnexpected
 
 sendContent :: Key -> Offset -> Proto Bool
 sendContent key offset = do
diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs
index 3c715fbde1..2caa7cdb1b 100644
--- a/RemoteDaemon/Transport/Tor.hs
+++ b/RemoteDaemon/Transport/Tor.hs
@@ -17,6 +17,7 @@ import Utility.FileMode
 import Utility.AuthToken
 import Remote.Helper.Tor
 import P2P.Protocol
+import P2P.IO
 import P2P.Annex
 import P2P.Auth
 import Annex.UUID
@@ -90,7 +91,12 @@ serveClient th u r q = bracket setup cleanup go
 				, runIhdl = h
 				, runOhdl = h
 				}
-			void $ runFullProto runenv (serve u)
+			v <- liftIO $ runNetProto runenv $ serveAuth u
+			case v of
+				Just (Just theiruuid) -> void $ 
+					runFullProto (Serving theiruuid) runenv $
+						serveAuthed u
+				_ -> return ()
 		-- Merge the duplicated state back in.
 		liftAnnex th $ mergeState st'
 		debugM "remotedaemon" "done with TOR connection"

From a8c868c2e1cde17379c4423c629378cf77c6ebfc Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 2 Dec 2016 16:39:01 -0400
Subject: [PATCH 157/367] plumb assicated files through P2P protocol for
 updating transfer logs

ReadContent can't update the log, since it reads lazily. This part of
the P2P monad will need to be rethought.

Associated files are heavily sanitized when received from a peer;
they could be an exploit vector.

This commit was sponsored by Jochen Bartl on Patreon.
---
 Annex/Notification.hs |  6 ++-
 P2P/Annex.hs          | 43 ++++++++++++--------
 P2P/Protocol.hs       | 94 ++++++++++++++++++++++++++++++-------------
 3 files changed, 98 insertions(+), 45 deletions(-)

diff --git a/Annex/Notification.hs b/Annex/Notification.hs
index 4f492878b9..e61b362adc 100644
--- a/Annex/Notification.hs
+++ b/Annex/Notification.hs
@@ -7,7 +7,7 @@
 
 {-# LANGUAGE CPP #-}
 
-module Annex.Notification (NotifyWitness, notifyTransfer, notifyDrop) where
+module Annex.Notification (NotifyWitness, noNotification, notifyTransfer, notifyDrop) where
 
 import Annex.Common
 import Types.Transfer
@@ -21,6 +21,10 @@ import qualified DBus.Client
 -- Witness that notification has happened.
 data NotifyWitness = NotifyWitness
 
+-- Only use when no notification should be done.
+noNotification :: NotifyWitness
+noNotification = NotifyWitness
+
 {- Wrap around an action that performs a transfer, which may run multiple
  - attempts. Displays notification when supported and when the user asked
  - for it. -}
diff --git a/P2P/Annex.hs b/P2P/Annex.hs
index dce4ceebac..5e1763fc6e 100644
--- a/P2P/Annex.hs
+++ b/P2P/Annex.hs
@@ -15,6 +15,8 @@ module P2P.Annex
 
 import Annex.Common
 import Annex.Content
+import Annex.Transfer
+import Annex.Notification
 import P2P.Protocol
 import P2P.IO
 import Logs.Location
@@ -48,8 +50,8 @@ runLocal runmode runner a = case a of
 		let getsize = liftIO . catchMaybeIO . getFileSize
 		size <- inAnnex' isJust Nothing getsize k
 		runner (next (Len <$> size))
-	-- TODO transfer logs
-	ReadContent k (Offset o) next -> do
+	-- TODO transfer log not updated
+	ReadContent k af (Offset o) next -> do
 		v <- tryNonAsync $ prepSendAnnex k
 		case v of
 			-- The check can detect a problem after the
@@ -57,25 +59,26 @@ runLocal runmode runner a = case a of
 			-- Instead, the receiving peer must AlwaysVerify
 			-- the content it receives.
 			Right (Just (f, _check)) -> do
-				v' <- liftIO $ tryNonAsync $ do
-					h <- openBinaryFile f ReadMode
-					when (o /= 0) $
-						hSeek h AbsoluteSeek o
-					L.hGetContents h
+				v' <- tryNonAsync $ -- transfer upload k af $
+					liftIO $ do
+						h <- openBinaryFile f ReadMode
+						when (o /= 0) $
+							hSeek h AbsoluteSeek o
+						L.hGetContents h
 				case v' of
 					Left _ -> return Nothing
 					Right b -> runner (next b)
 			_ -> return Nothing
-	-- TODO transfer logs
-	WriteContent k (Offset o) (Len l) b next -> do
+	WriteContent k af (Offset o) (Len l) b next -> do
 		ok <- flip catchNonAsync (const $ return False) $
-			getViaTmp AlwaysVerify k $ \tmp -> liftIO $ do
-				withBinaryFile tmp WriteMode $ \h -> do
-					when (o /= 0) $
-						hSeek h AbsoluteSeek o
-					L.hPut h b
-				sz <- getFileSize tmp
-				return (toInteger sz == l, UnVerified)
+			transfer download k af $
+				getViaTmp AlwaysVerify k $ \tmp -> liftIO $ do
+					withBinaryFile tmp WriteMode $ \h -> do
+						when (o /= 0) $
+							hSeek h AbsoluteSeek o
+						L.hPut h b
+					sz <- getFileSize tmp
+					return (toInteger sz == l, UnVerified)
 		runner (next ok)
 	SetPresent k u next -> do
 		v <- tryNonAsync $ logChange k u InfoPresent
@@ -107,3 +110,11 @@ runLocal runmode runner a = case a of
 				protoaction False
 				next
 			Right _ -> runner next
+  where
+	transfer mk k af a = case runmode of
+		-- Update transfer logs when serving.
+		Serving theiruuid -> 
+			mk theiruuid k af noRetry (const a) noNotification
+		-- Transfer logs are updated higher in the stack when
+		-- a client.
+		Client -> a
diff --git a/P2P/Protocol.hs b/P2P/Protocol.hs
index 9678b7954b..53c3265efe 100644
--- a/P2P/Protocol.hs
+++ b/P2P/Protocol.hs
@@ -5,7 +5,9 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-{-# LANGUAGE DeriveFunctor, TemplateHaskell, FlexibleContexts, RankNTypes #-}
+{-# LANGUAGE DeriveFunctor, TemplateHaskell, FlexibleContexts #-}
+{-# LANGUAGE TypeSynonymInstances, FlexibleInstances, RankNTypes #-}
+{-# OPTIONS_GHC -fno-warn-orphans #-}
 
 module P2P.Protocol where
 
@@ -15,14 +17,17 @@ import Types.UUID
 import Utility.AuthToken
 import Utility.Applicative
 import Utility.PartialPrelude
+import Git.FilePath
 
 import Control.Monad
 import Control.Monad.Free
 import Control.Monad.Free.TH
 import Control.Monad.Catch
+import System.FilePath
 import System.Exit (ExitCode(..))
 import System.IO
 import qualified Data.ByteString.Lazy as L
+import Data.Char
 
 newtype Offset = Offset Integer
 	deriving (Show)
@@ -46,8 +51,8 @@ data Message
 	| LOCKCONTENT Key
 	| UNLOCKCONTENT
 	| REMOVE Key
-	| GET Offset Key
-	| PUT Key
+	| GET Offset AssociatedFile Key
+	| PUT AssociatedFile Key
 	| PUT_FROM Offset
 	| ALREADY_HAVE
 	| SUCCESS
@@ -66,8 +71,8 @@ instance Proto.Sendable Message where
 	formatMessage (LOCKCONTENT key) = ["LOCKCONTENT", Proto.serialize key]
 	formatMessage UNLOCKCONTENT = ["UNLOCKCONTENT"]
 	formatMessage (REMOVE key) = ["REMOVE", Proto.serialize key]
-	formatMessage (GET offset key) = ["GET", Proto.serialize offset, Proto.serialize key]
-	formatMessage (PUT key) = ["PUT", Proto.serialize key]
+	formatMessage (GET offset af key) = ["GET", Proto.serialize offset, Proto.serialize af, Proto.serialize key]
+	formatMessage (PUT af key) = ["PUT", Proto.serialize af, Proto.serialize key]
 	formatMessage (PUT_FROM offset) = ["PUT-FROM", Proto.serialize offset]
 	formatMessage ALREADY_HAVE = ["ALREADY-HAVE"]
 	formatMessage SUCCESS = ["SUCCESS"]
@@ -85,8 +90,8 @@ instance Proto.Receivable Message where
 	parseCommand "LOCKCONTENT" = Proto.parse1 LOCKCONTENT
 	parseCommand "UNLOCKCONTENT" = Proto.parse0 UNLOCKCONTENT
 	parseCommand "REMOVE" = Proto.parse1 REMOVE
-	parseCommand "GET" = Proto.parse2 GET
-	parseCommand "PUT" = Proto.parse1 PUT
+	parseCommand "GET" = Proto.parse3 GET
+	parseCommand "PUT" = Proto.parse2 PUT
 	parseCommand "PUT-FROM" = Proto.parse1 PUT_FROM
 	parseCommand "ALREADY-HAVE" = Proto.parse0 ALREADY_HAVE
 	parseCommand "SUCCESS" = Proto.parse0 SUCCESS
@@ -110,6 +115,38 @@ instance Proto.Serializable Service where
 	deserialize "git-receive-pack" = Just ReceivePack
 	deserialize _ = Nothing
 
+-- | Since AssociatedFile is not the last thing in a protocol line,
+-- its serialization cannot contain any whitespace. This is handled
+-- by replacing whitespace with '%' (and '%' with '%%')
+--
+-- When deserializing an AssociatedFile from a peer, it's sanitized,
+-- to avoid any unusual characters that might cause problems when it's
+-- displayed to the user.
+--
+-- These mungings are ok, because an AssociatedFile is only ever displayed
+-- to the user and does not need to match a file on disk.
+instance Proto.Serializable AssociatedFile where
+	serialize Nothing = ""
+	serialize (Just af) = toInternalGitPath $ concatMap esc af
+	  where
+		esc '%' = "%%"
+		esc c 
+			| isSpace c = "%"
+			| otherwise = [c]
+	
+	deserialize s = case fromInternalGitPath $ deesc [] s of
+		[] -> Just Nothing
+		f
+			| isRelative f -> Just (Just f)
+			| otherwise -> Nothing
+	  where
+	  	deesc b [] = reverse b
+		deesc b ('%':'%':cs) = deesc ('%':b) cs
+		deesc b ('%':cs) = deesc ('_':b) cs
+		deesc b (c:cs)
+			| isControl c = deesc ('_':b) cs
+			| otherwise = deesc (c:b) cs
+
 -- | Free monad for the protocol, combining net communication,
 -- and local actions.
 data ProtoF c = Net (NetF c) | Local (LocalF c)
@@ -155,10 +192,10 @@ data LocalF c
 	| ContentSize Key (Maybe Len -> c)
 	-- ^ Gets size of the content of a key, when the full content is
 	-- present.
-	| ReadContent Key Offset (L.ByteString -> c)
+	| ReadContent Key AssociatedFile Offset (L.ByteString -> c)
 	-- ^ Lazily reads the content of a key. Note that the content
 	-- may change while it's being sent.
-	| WriteContent Key Offset Len L.ByteString (Bool -> c)
+	| WriteContent Key AssociatedFile Offset Len L.ByteString (Bool -> c)
 	-- ^ Writes content to temp file starting at an offset.
 	-- Once the whole content of the key has been stored, moves the
 	-- temp file into place and returns True.
@@ -226,15 +263,15 @@ remove key = do
 	net $ sendMessage (REMOVE key)
 	checkSuccess
 
-get :: Key -> Proto Bool
-get key = receiveContent key (`GET` key)
+get :: Key -> AssociatedFile -> Proto Bool
+get key af = receiveContent key af (\offset -> GET offset af key)
 
-put :: Key -> Proto Bool
-put key = do
-	net $ sendMessage (PUT key)
+put :: Key -> AssociatedFile -> Proto Bool
+put key af = do
+	net $ sendMessage (PUT af key)
 	r <- net receiveMessage
 	case r of
-		PUT_FROM offset -> sendContent key offset
+		PUT_FROM offset -> sendContent key af offset
 		ALREADY_HAVE -> return True
 		_ -> do
 			net $ sendMessage (ERROR "expected PUT_FROM")
@@ -307,17 +344,17 @@ serveAuthed myuuid = void $ serverLoop handler
 	handler (REMOVE key) = do
 		sendSuccess =<< local (removeContent key)
 		return ServerContinue
-	handler (PUT key) = do
+	handler (PUT af key) = do
 		have <- local $ checkContentPresent key
 		if have
 			then net $ sendMessage ALREADY_HAVE
 			else do
-				ok <- receiveContent key PUT_FROM
+				ok <- receiveContent key af PUT_FROM
 				when ok $
 					local $ setPresent key myuuid
 		return ServerContinue
-	handler (GET offset key) = do
-		void $ sendContent key offset
+	handler (GET offset key af) = do
+		void $ sendContent af key offset
 		-- setPresent not called because the peer may have
 		-- requested the data but not permanently stored it.
 		return ServerContinue
@@ -326,22 +363,22 @@ serveAuthed myuuid = void $ serverLoop handler
 		return ServerContinue
 	handler _ = return ServerUnexpected
 
-sendContent :: Key -> Offset -> Proto Bool
-sendContent key offset = do
-	(len, content) <- readContentLen key offset
+sendContent :: Key -> AssociatedFile -> Offset -> Proto Bool
+sendContent key af offset = do
+	(len, content) <- readContentLen key af offset
 	net $ sendMessage (DATA len)
 	net $ sendBytes len content
 	checkSuccess
 
-receiveContent :: Key -> (Offset -> Message) -> Proto Bool
-receiveContent key mkmsg = do
+receiveContent :: Key -> AssociatedFile -> (Offset -> Message) -> Proto Bool
+receiveContent key af mkmsg = do
 	Len n <- local $ tmpContentSize key
 	let offset = Offset n
 	net $ sendMessage (mkmsg offset)
 	r <- net receiveMessage
 	case r of
 		DATA len -> do
-			ok <- local . writeContent key offset len
+			ok <- local . writeContent key af offset len
 				=<< net (receiveBytes len)
 			sendSuccess ok
 			return ok
@@ -366,8 +403,8 @@ sendSuccess False = net $ sendMessage FAILURE
 -- Reads content from an offset. The Len should correspond to
 -- the length of the ByteString, but to avoid buffering the content
 -- in memory, is gotten using contentSize.
-readContentLen :: Key -> Offset -> Proto (Len, L.ByteString)
-readContentLen key (Offset offset) = go =<< local (contentSize key)
+readContentLen :: Key -> AssociatedFile -> Offset -> Proto (Len, L.ByteString)
+readContentLen key af (Offset offset) = go =<< local (contentSize key)
   where
 	go Nothing = return (Len 0, L.empty)
 	go (Just (Len totallen)) = do
@@ -375,7 +412,8 @@ readContentLen key (Offset offset) = go =<< local (contentSize key)
 		if len <= 0
 			then return (Len 0, L.empty)
 			else do
-				content <- local $ readContent key (Offset offset)
+				content <- local $
+					readContent key af (Offset offset)
 				return (Len len, content)
 
 connect :: Service -> Handle -> Handle -> Proto ExitCode

From 67bc5c2e35db4e3029a817c2dbb1080ba94e0839 Mon Sep 17 00:00:00 2001
From: "lorenzo@a77fdbe826df1c175cbd30ed050312ccf7b7fc7b" <lorenzo@web>
Date: Sat, 3 Dec 2016 09:04:45 +0000
Subject: [PATCH 158/367]

---
 doc/todo/Long_Running_Filter_Process.mdwn | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 doc/todo/Long_Running_Filter_Process.mdwn

diff --git a/doc/todo/Long_Running_Filter_Process.mdwn b/doc/todo/Long_Running_Filter_Process.mdwn
new file mode 100644
index 0000000000..329abaf453
--- /dev/null
+++ b/doc/todo/Long_Running_Filter_Process.mdwn
@@ -0,0 +1,22 @@
+Hello,
+
+while reading the release notes of git 2.11 I noticed a cool new feature has been merged:
+
+> If the filter command (a string value) is defined via
+> `filter.<driver>.process` then Git can process all blobs with a
+> single filter invocation for the entire life of a single Git
+> command.
+
+see the [git documentation][1].
+
+This has been developed in the context of git-lfs (see [PR 1382] [2]).
+
+If I understand correctly how it works this could speed up v6 repos. Looking at the history/website 
+of git-annex there doesn't seem to be yet any work on this so I though it was worth calling the 
+attention on the feature.
+
+Thanks a lot for all the work on git-annex, it's a really amazing project! The more I study it the more cool features I discover :)
+
+
+[1]: https://github.com/git/git/blob/v2.11.0/Documentation/gitattributes.txt#L384
+[2]: https://github.com/git-lfs/git-lfs/pull/1382

From ae0f60b62880bde5dace93273b11f57a35f27e86 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 3 Dec 2016 16:30:48 -0400
Subject: [PATCH 159/367] break out curent list of names into its own file so I
 can auto-add from patreon

and some misc updates
---
 doc/thanks/list | 44 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100644 doc/thanks/list

diff --git a/doc/thanks/list b/doc/thanks/list
new file mode 100644
index 0000000000..7b5a19ec1e
--- /dev/null
+++ b/doc/thanks/list
@@ -0,0 +1,44 @@
+martin f. krafft, 
+John Byrnes, 
+Aurélien Couderc, 
+Jeffrey Chiu, 
+Amitai Schlair, 
+Andreas, 
+Anthony DeRobertis, 
+Baldur Kristinsson, 
+Boyd Stephen Smith Jr., 
+Brock Spratlen, 
+Christian Diller, 
+Denis Dzyubenko, 
+Eskild Hustvedt, 
+Evgeni Kunev, 
+FBC, 
+Fernando Jimenez, 
+Greg Young, 
+Henrik RIomar, 
+Ignacio, 
+Jake Vosloo, 
+James Valleroy, 
+Jason Woofenden, 
+Jeff Goeke-Smith, 
+Jim, 
+Jochen Bartl, 
+Johannes Schlatow, 
+John Carr, 
+Josh Taylor, 
+Josh Triplett, 
+Kuno Woudt, 
+Matthias Urlichs, 
+Mattias J, 
+Nathan Howell, 
+Nick Daly, 
+Nicolas Schodet, 
+Ole-Morten Duesund, 
+Remy van Elst, 
+RémiV, 
+Thom May, 
+Thomas Ferris Nicolaisen, 
+Thomas Hochstein, 
+Tyler Cipriani, 
+encryptio, 
+Øyvind A. Holm,

From 1a4f9660c8085ff81d7e5d6cc5e976ce47098882 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 3 Dec 2016 16:51:16 -0400
Subject: [PATCH 160/367] update

---
 doc/thanks/list | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/doc/thanks/list b/doc/thanks/list
index 7b5a19ec1e..653466f814 100644
--- a/doc/thanks/list
+++ b/doc/thanks/list
@@ -41,4 +41,13 @@ Thomas Ferris Nicolaisen,
 Thomas Hochstein, 
 Tyler Cipriani, 
 encryptio, 
-Øyvind A. Holm,
+Øyvind A. Holm, 
+Bruno BEAUFILS, 
+Rémi Vanicat, 
+Trenton Cronholm, 
+Francois Marier, 
+Peter Hogg, 
+Amitai Schleier, 
+Brennen Bearnes, 
+Tim Howes, 
+Willard Korfhage, 

From 6d39a30d79d49134a4e528f2838b3ec89fdcf676 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 3 Dec 2016 16:52:18 -0400
Subject: [PATCH 161/367] use list

---
 doc/thanks.mdwn | 22 ++++++----------------
 1 file changed, 6 insertions(+), 16 deletions(-)

diff --git a/doc/thanks.mdwn b/doc/thanks.mdwn
index 645cca732a..eec4686066 100644
--- a/doc/thanks.mdwn
+++ b/doc/thanks.mdwn
@@ -1,6 +1,6 @@
 The development of git-annex was made possible by the generous
 donations of many people. I want to say "Thank You!" to each of
-you individually, but until I meet all 1500 of you, this page will have to
+you individually, but until I meet all 1500+ of you, this page will have to
 do. You have my most sincere thanks. --[[Joey]]
 
 You can support my git-annex development
@@ -18,16 +18,8 @@ git-annex development is partially supported by the
 [NSF](https://www.nsf.gov/awardsearch/showAward?AWD_ID=1429999) as a part of the
 [DataLad project](http://datalad.org/).
 
-Thanks also to these folks for their support: martin f. krafft, John Byrnes,
-Aurélien Couderc, Jeffrey Chiu, Amitai Schlair, Andreas, Anthony DeRobertis,
-Baldur Kristinsson, Boyd Stephen Smith Jr., Brock Spratlen, Christian Diller,
-Denis Dzyubenko, Eskild Hustvedt, Evgeni Kunev, FBC, Fernando Jimenez, Greg
-Young, Henrik RIomar, Ignacio, Jake Vosloo, James Valleroy, Jason Woofenden,
-Jeff Goeke-Smith, Jim, Jochen Bartl, Johannes Schlatow, John Carr, Josh
-Taylor, Josh Triplett, Kuno Woudt, Matthias Urlichs, Mattias J, Nathan Howell,
-Nick Daly, Nicolas Schodet, Ole-Morten Duesund, Remy van Elst, RémiV, Thom
-May, Thomas Ferris Nicolaisen, Thomas Hochstein, Tyler Cipriani, encryptio,
-Øyvind A. Holm
+Thanks also to these folks for their support: 
+[[!inline raw=yes pages="thanks/list"]] and anonymous supporters.
 
 ## 2013-2014
 
@@ -385,13 +377,11 @@ Tyree, Aaron Whitehouse
 * Rsync.net, for providing me a free account so I can make sure git-annex
   works well with it.
 * LeastAuthority.com, for providing me a free Tahoe-LAFS grid account,
-  so I can test git-annex with that, and back up the git-annex assistant
-  screencasts.
+  so I can test git-annex with that.
+* Yury V. Zaytsev for running the Windows autobuilder.
+* Kevin McKenzie for providing a OSX account for testing.
 * Anna and Mark, for the loan of the video camera; as well as the rest of
   my family, for your support. Even when I couldn't explain what I was
   working on.
-* The Hodges, for providing such a congenial place for me to live and work
-  on these first world problems, while you're off helping people in the
-  third world.
 * And Mom, for stamping and stuffing so many thank you envelopes, and all the
   rhubarb pies.

From f69a79fc70773f8b81e1586be4a476018cc276fc Mon Sep 17 00:00:00 2001
From: "justin.lebar@7a36fcafc322d9a381e89f08ab6289033c6dde91"
 <justinlebar@web>
Date: Sun, 4 Dec 2016 04:30:38 +0000
Subject: [PATCH 162/367] Added a comment

---
 ...omment_3_5ac676877feaa7cdb9e05d6b71b1a4c3._comment | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 doc/bugs/Nearline_bucket_stopped_working___40__can__39__t_even_HEAD_files__41__/comment_3_5ac676877feaa7cdb9e05d6b71b1a4c3._comment

diff --git a/doc/bugs/Nearline_bucket_stopped_working___40__can__39__t_even_HEAD_files__41__/comment_3_5ac676877feaa7cdb9e05d6b71b1a4c3._comment b/doc/bugs/Nearline_bucket_stopped_working___40__can__39__t_even_HEAD_files__41__/comment_3_5ac676877feaa7cdb9e05d6b71b1a4c3._comment
new file mode 100644
index 0000000000..67b215bc72
--- /dev/null
+++ b/doc/bugs/Nearline_bucket_stopped_working___40__can__39__t_even_HEAD_files__41__/comment_3_5ac676877feaa7cdb9e05d6b71b1a4c3._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="justin.lebar@7a36fcafc322d9a381e89f08ab6289033c6dde91"
+ nickname="justin.lebar"
+ avatar="http://cdn.libravatar.org/avatar/9fca4b61a1ab555f231851e7543f9a3e"
+ subject="comment 3"
+ date="2016-12-04T04:30:38Z"
+ content="""
+For a while things were working, but now it's not working again, same problem as before.
+
+Do you think maybe it's a timestamp bug in the signature or something?  That could explain this \"mysteriously works then stops working\" behavior.
+"""]]

From 19b1c5bf598b4f51320550309fb1757480b93ccd Mon Sep 17 00:00:00 2001
From: "marek@33e8ba4fbc201af14a2badcc0656024401f5c916" <marek@web>
Date: Sun, 4 Dec 2016 15:28:41 +0000
Subject: [PATCH 163/367]

---
 doc/forum/more_intelligent_copy_.mdwn | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/forum/more_intelligent_copy_.mdwn

diff --git a/doc/forum/more_intelligent_copy_.mdwn b/doc/forum/more_intelligent_copy_.mdwn
new file mode 100644
index 0000000000..1c9889a74e
--- /dev/null
+++ b/doc/forum/more_intelligent_copy_.mdwn
@@ -0,0 +1,15 @@
+Hi,
+
+I noticed, that 
+
+git annex copy --to REMOTE FILES
+
+and 
+
+git annex copy --to REMOTE --not --in REMOTE FILES
+
+behave differently. The first does not check, whether file contents are already in the remote the latter does that. I realize that this mimics "normal" (UNIX) copy behaviour but I was not entirely certain this was desired.
+Depending on the type of the remote and its configuration (encryption) the latter is considerably faster.
+
+Just my two cents.
+

From 3b7a51ea0eab2ce89c9a4d7d54125413dbf3fe6f Mon Sep 17 00:00:00 2001
From: "https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
 <https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4@web>
Date: Sun, 4 Dec 2016 15:52:29 +0000
Subject: [PATCH 164/367] initial report

---
 ...ex_add_ignores_.-prefixed_directories.mdwn | 74 +++++++++++++++++++
 1 file changed, 74 insertions(+)
 create mode 100644 doc/bugs/annex_add_ignores_.-prefixed_directories.mdwn

diff --git a/doc/bugs/annex_add_ignores_.-prefixed_directories.mdwn b/doc/bugs/annex_add_ignores_.-prefixed_directories.mdwn
new file mode 100644
index 0000000000..f4ebfe6d20
--- /dev/null
+++ b/doc/bugs/annex_add_ignores_.-prefixed_directories.mdwn
@@ -0,0 +1,74 @@
+### Please describe the problem.
+
+annex add  seems to ignore content under directories having . prefix.
+
+We thought to unify (across direct/indirect/v6) adding files to annex repository by using 'git annex add' with corresponding setting for largefiles for any addition, but it seems to ignore content under .-prefixed directories, unlike git
+
+### What version of git-annex are you using? On what operating system?
+
+6.20161122+gitg9f179ae-1~ndall+1
+
+### Please provide any additional information below.
+
+[[!format sh """
+hopa:/tmp/datalad_temp_test_annex_add_no_dotfilesqMXck8
+$> git status                      
+On branch master
+ 
+Initial commit
+
+nothing to commit (create/copy files and use "git add" to track)
+
+$> mkdir .dir dir; echo 123 > .dir/123; echo 124 > dir/124
+
+$> git status
+On branch master
+
+Initial commit
+
+Untracked files:
+  (use "git add <file>..." to include in what will be committed)
+
+	.dir/
+	dir/
+
+nothing added to commit but untracked files present (use "git add" to track)
+
+$> git annex add -c 'annex.largefiles=nothing' .
+add dir/124 (non-large file; adding content to git repository) ok
+(recording state in git...)
+
+$> git status
+On branch master
+
+Initial commit
+
+Changes to be committed:
+  (use "git rm --cached <file>..." to unstage)
+
+	new file:   dir/124
+
+Untracked files:
+  (use "git add <file>..." to include in what will be committed)
+
+	.dir/
+
+
+# and with regular git
+$> git  -c 'annex.largefiles=nothing' add . 
+
+$> git status
+On branch master
+
+Initial commit
+
+Changes to be committed:
+  (use "git rm --cached <file>..." to unstage)
+
+	new file:   .dir/123
+	new file:   dir/124
+
+
+"""]]
+
+Ref: https://github.com/datalad/datalad/issues/1027

From 8cb409150cc3525800111bf48486ab90df7d1976 Mon Sep 17 00:00:00 2001
From: "https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
 <https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4@web>
Date: Sun, 4 Dec 2016 15:54:32 +0000
Subject: [PATCH 165/367] added forgotten author tag

---
 doc/bugs/annex_add_ignores_.-prefixed_directories.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/annex_add_ignores_.-prefixed_directories.mdwn b/doc/bugs/annex_add_ignores_.-prefixed_directories.mdwn
index f4ebfe6d20..66f3b654f2 100644
--- a/doc/bugs/annex_add_ignores_.-prefixed_directories.mdwn
+++ b/doc/bugs/annex_add_ignores_.-prefixed_directories.mdwn
@@ -72,3 +72,5 @@ Changes to be committed:
 """]]
 
 Ref: https://github.com/datalad/datalad/issues/1027
+
+[[!meta author=yoh]]

From e0be5b354ac800dc8e8da152427778ae305700cd Mon Sep 17 00:00:00 2001
From: "https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4"
 <https://me.yahoo.com/a/EbvxpTI_xP9Aod7Mg4cwGhgjrCrdM5s-#7c0f4@web>
Date: Sun, 4 Dec 2016 16:20:00 +0000
Subject: [PATCH 166/367] "fixed" by reading --help

---
 doc/bugs/annex_add_ignores_.-prefixed_directories.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/annex_add_ignores_.-prefixed_directories.mdwn b/doc/bugs/annex_add_ignores_.-prefixed_directories.mdwn
index 66f3b654f2..5db40f4dd6 100644
--- a/doc/bugs/annex_add_ignores_.-prefixed_directories.mdwn
+++ b/doc/bugs/annex_add_ignores_.-prefixed_directories.mdwn
@@ -74,3 +74,5 @@ Changes to be committed:
 Ref: https://github.com/datalad/datalad/issues/1027
 
 [[!meta author=yoh]]
+
+[[done]]; oh -- it is RTFM: --include-dotfiles  --[[yoh]]

From 24317be64669949249a2fd3fd8f1cede6bdff4b9 Mon Sep 17 00:00:00 2001
From: Daniel Brooks <db48x@db48x.net>
Date: Mon, 5 Dec 2016 09:59:20 -0500
Subject: [PATCH 167/367] git-annex fromkey now takes multiple pairs of keys
 and filenames

It also still reads from stdin when none are specified.
---
 Command/FromKey.hs         | 15 +++++++++------
 doc/git-annex-fromkey.mdwn |  8 +++++---
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index 670e9e6a6b..0916cf79dd 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -20,16 +20,18 @@ import Network.URI
 cmd :: Command
 cmd = notDirect $ notBareRepo $
 	command "fromkey" SectionPlumbing "adds a file using a specific key"
-		(paramPair paramKey paramPath)
+		(paramRepeating (paramPair paramKey paramPath))
 		(withParams seek)
 
 seek :: CmdParams -> CommandSeek
+seek [] = do
+	withNothing startMass []
 seek ps = do
 	force <- Annex.getState Annex.force
-	withWords (start force) ps
+	withPairs (start force) ps
 
-start :: Bool -> [String] -> CommandStart
-start force (keyname:file:[]) = do
+start :: Bool -> (String, FilePath) -> CommandStart
+start force (keyname, file) = do
 	let key = mkKey keyname
 	unless force $ do
 		inbackend <- inAnnex key
@@ -37,10 +39,11 @@ start force (keyname:file:[]) = do
 			"key ("++ keyname ++") is not present in backend (use --force to override this sanity check)"
 	showStart "fromkey" file
 	next $ perform key file
-start _ [] = do
+
+startMass :: CommandStart
+startMass = do
 	showStart "fromkey" "stdin"
 	next massAdd
-start _ _ = giveup "specify a key and a dest file"
 
 massAdd :: CommandPerform
 massAdd = go True =<< map (separate (== ' ')) . lines <$> liftIO getContents
diff --git a/doc/git-annex-fromkey.mdwn b/doc/git-annex-fromkey.mdwn
index 461f42eb6b..2591e97859 100644
--- a/doc/git-annex-fromkey.mdwn
+++ b/doc/git-annex-fromkey.mdwn
@@ -4,14 +4,16 @@ git-annex fromkey - adds a file using a specific key
 
 # SYNOPSIS
 
-git annex fromkey `[key file]`
+git annex fromkey `[key file ...]`
 
 # DESCRIPTION
 
 This plumbing-level command can be used to manually set up a file
 in the git repository to link to a specified key.
 
-If the key and file are not specified on the command line, they are
+Multiple pairs of file and key can be given in a single command line.
+
+If no key and file pair are specified on the command line, they are
 instead read from stdin. Any number of lines can be provided in this
 mode, each containing a key and filename, separated by a single space.
 
@@ -26,7 +28,7 @@ to do that.
 * `--force`
 
   Allow making a file link to a key whose content is not in the local
-  repository. The key may not be known to git-annex at all.  
+  repository. The key may not be known to git-annex at all.
 
 # SEE ALSO
 

From 93852dd7e8403e69c1d7b1334609c016202b0f46 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 5 Dec 2016 12:10:07 -0400
Subject: [PATCH 168/367] rmurl: --batch

* rmurl: Multiple pairs of files and urls can be provided on the
  command line.
* rmurl: Added --batch mode.

This commit was sponsored by Trenton Cronholm on Patreon.
---
 CHANGELOG                |  3 +++
 Command/RmUrl.hs         | 33 +++++++++++++++++++++++++++------
 doc/git-annex-rmurl.mdwn | 14 +++++++++++++-
 3 files changed, 43 insertions(+), 7 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index 76da79eaaf..384dc3c3bc 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -8,6 +8,9 @@ git-annex (6.20161119) UNRELEASED; urgency=medium
     does not support graphical display, while xdot does.
   * Debian: xdot is a better interactive viewer than dot, so Suggest
     xdot, rather than graphviz.
+  * rmurl: Multiple pairs of files and urls can be provided on the
+    command line.
+  * rmurl: Added --batch mode.
 
  -- Joey Hess <id@joeyh.name>  Mon, 21 Nov 2016 11:27:50 -0400
 
diff --git a/Command/RmUrl.hs b/Command/RmUrl.hs
index eb78f7ba7e..0f33d66b2d 100644
--- a/Command/RmUrl.hs
+++ b/Command/RmUrl.hs
@@ -1,6 +1,6 @@
 {- git-annex command
  -
- - Copyright 2013 Joey Hess <id@joeyh.name>
+ - Copyright 2013-2016 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -10,18 +10,39 @@ module Command.RmUrl where
 import Command
 import Logs.Web
 import qualified Remote
+import CmdLine.Batch
 
 cmd :: Command
 cmd = notBareRepo $
 	command "rmurl" SectionCommon 
 		"record file is not available at url"
-		(paramPair paramFile paramUrl)
-		(withParams seek)
+		(paramRepeating (paramPair paramFile paramUrl))
+		(seek <$$> optParser)
 
-seek :: CmdParams -> CommandSeek
-seek = withPairs start
+data RmUrlOptions = RmUrlOptions
+	{ rmThese :: CmdParams
+	, batchOption :: BatchMode
+	}
 
-start :: (FilePath, String) -> CommandStart
+optParser :: CmdParamsDesc -> Parser RmUrlOptions
+optParser desc = RmUrlOptions
+	<$> cmdParams desc
+	<*> parseBatchOption
+
+seek :: RmUrlOptions -> CommandSeek
+seek o = case batchOption o of
+	Batch -> batchInput batchParser (batchCommandAction . start)
+	NoBatch -> withPairs start (rmThese o)
+
+-- Split on the last space, since a FilePath can contain whitespace,
+-- but a url should not.
+batchParser :: String -> Either String (FilePath, URLString)
+batchParser s = case separate (== ' ') (reverse s) of
+	(ru, rf)
+		| null ru || null rf -> Left "Expected: \"file url\""
+		| otherwise -> Right (reverse rf, reverse ru)
+
+start :: (FilePath, URLString) -> CommandStart
 start (file, url) = flip whenAnnexed file $ \_ key -> do
 	showStart "rmurl" file
 	next $ next $ cleanup url key
diff --git a/doc/git-annex-rmurl.mdwn b/doc/git-annex-rmurl.mdwn
index 5faf9ea39e..e971e622e6 100644
--- a/doc/git-annex-rmurl.mdwn
+++ b/doc/git-annex-rmurl.mdwn
@@ -4,12 +4,24 @@ git-annex rmurl - record file is not available at url
 
 # SYNOPSIS
 
-git annex rmurl `file url`
+git annex rmurl `[file url ..]`
 
 # DESCRIPTION
 
 Record that the file is no longer available at the url.
 
+If nothing is specified on the command line, they are instead read
+from stdin. Any number of lines can be provided in this mode,
+each containing a file and and url, separated by a single space.
+
+# OPTIONS
+
+* `--batch`
+
+  Enables batch mode, in which lines are read from stdin.
+  Each line should contain the file, and the url to remove from that file,
+  separated by a single space.
+
 # SEE ALSO
 
 [[git-annex]](1)

From 702c671c92aaa6fae05d80e40eb863cf95b92a10 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 5 Dec 2016 12:13:25 -0400
Subject: [PATCH 169/367] cleanup

---
 doc/git-annex-rmurl.mdwn | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/doc/git-annex-rmurl.mdwn b/doc/git-annex-rmurl.mdwn
index e971e622e6..504685a582 100644
--- a/doc/git-annex-rmurl.mdwn
+++ b/doc/git-annex-rmurl.mdwn
@@ -10,10 +10,6 @@ git annex rmurl `[file url ..]`
 
 Record that the file is no longer available at the url.
 
-If nothing is specified on the command line, they are instead read
-from stdin. Any number of lines can be provided in this mode,
-each containing a file and and url, separated by a single space.
-
 # OPTIONS
 
 * `--batch`

From 6246c4a6dbf1a26d67834ea24e0b8d4b8c9917e6 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 5 Dec 2016 12:16:07 -0400
Subject: [PATCH 170/367] minor style

---
 Command/FromKey.hs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index 0916cf79dd..dca63aabe2 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -24,8 +24,7 @@ cmd = notDirect $ notBareRepo $
 		(withParams seek)
 
 seek :: CmdParams -> CommandSeek
-seek [] = do
-	withNothing startMass []
+seek [] = withNothing startMass []
 seek ps = do
 	force <- Annex.getState Annex.force
 	withPairs (start force) ps

From e65c31e56b2d7db51b2134f1f56d1617ec049c2a Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 5 Dec 2016 12:16:35 -0400
Subject: [PATCH 171/367] changelog

---
 CHANGELOG | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CHANGELOG b/CHANGELOG
index 384dc3c3bc..215349f4bd 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -11,6 +11,8 @@ git-annex (6.20161119) UNRELEASED; urgency=medium
   * rmurl: Multiple pairs of files and urls can be provided on the
     command line.
   * rmurl: Added --batch mode.
+  * fromkey: Accept multiple pairs of files and keys.
+    Thanks, Daniel Brooks.
 
  -- Joey Hess <id@joeyh.name>  Mon, 21 Nov 2016 11:27:50 -0400
 

From 82d01f56196f7a38dc0c56825fb36cd5683f690d Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 5 Dec 2016 12:55:50 -0400
Subject: [PATCH 172/367] rekey: Added --batch mode.

Would have liked to make the Parser parse the file and key pairs, but it
seems that optparse-applicative is unable to handle eg:

	many ((,) <$> argument <*> argument)

This commit was sponsored by Thomas Hochstein on Patreon.
---
 CHANGELOG                |  1 +
 Command/ReKey.hs         | 36 ++++++++++++++++++++++++++++++------
 Command/RmUrl.hs         |  1 -
 doc/git-annex-rekey.mdwn |  6 ++++++
 4 files changed, 37 insertions(+), 7 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index 215349f4bd..c30447f4e2 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -13,6 +13,7 @@ git-annex (6.20161119) UNRELEASED; urgency=medium
   * rmurl: Added --batch mode.
   * fromkey: Accept multiple pairs of files and keys.
     Thanks, Daniel Brooks.
+  * rekey: Added --batch mode.
 
  -- Joey Hess <id@joeyh.name>  Mon, 21 Nov 2016 11:27:50 -0400
 
diff --git a/Command/ReKey.hs b/Command/ReKey.hs
index 33734ebe79..4ddbd68b61 100644
--- a/Command/ReKey.hs
+++ b/Command/ReKey.hs
@@ -25,15 +25,39 @@ cmd = notDirect $
 	command "rekey" SectionPlumbing
 		"change keys used for files"
 		(paramRepeating $ paramPair paramPath paramKey)
-		(withParams seek)
+		(seek <$$> optParser)
 
-seek :: CmdParams -> CommandSeek
-seek = withPairs start
+data ReKeyOptions = ReKeyOptions
+	{ reKeyThese :: CmdParams
+	, batchOption :: BatchMode
+	}
 
-start :: (FilePath, String) -> CommandStart
-start (file, keyname) = ifAnnexed file go stop
+optParser :: CmdParamsDesc -> Parser ReKeyOptions
+optParser desc = ReKeyOptions
+	<$> cmdParams desc
+	<*> parseBatchOption
+
+-- Split on the last space, since a FilePath can contain whitespace,
+-- but a Key very rarely does.
+batchParser :: String -> Either String (FilePath, Key)
+batchParser s = case separate (== ' ') (reverse s) of
+	(rk, rf)
+		| null rk || null rf -> Left "Expected: \"file key\""
+		| otherwise -> case file2key (reverse rk) of
+			Nothing -> Left "bad key"
+			Just k -> Right (reverse rf, k)
+
+seek :: ReKeyOptions -> CommandSeek
+seek o = case batchOption o of
+	Batch -> batchInput batchParser (batchCommandAction . start)
+	NoBatch -> withPairs (start . parsekey) (reKeyThese o)
+  where
+	parsekey (file, skey) =
+		(file, fromMaybe (giveup "bad key") (file2key skey))
+
+start :: (FilePath, Key) -> CommandStart
+start (file, newkey) = ifAnnexed file go stop
   where
-	newkey = fromMaybe (giveup "bad key") $ file2key keyname
 	go oldkey
 		| oldkey == newkey = stop
 		| otherwise = do
diff --git a/Command/RmUrl.hs b/Command/RmUrl.hs
index 0f33d66b2d..1a547a71e2 100644
--- a/Command/RmUrl.hs
+++ b/Command/RmUrl.hs
@@ -10,7 +10,6 @@ module Command.RmUrl where
 import Command
 import Logs.Web
 import qualified Remote
-import CmdLine.Batch
 
 cmd :: Command
 cmd = notBareRepo $
diff --git a/doc/git-annex-rekey.mdwn b/doc/git-annex-rekey.mdwn
index 4ec0b49e84..ce5e43d419 100644
--- a/doc/git-annex-rekey.mdwn
+++ b/doc/git-annex-rekey.mdwn
@@ -20,6 +20,12 @@ Multiple pairs of file and key can be given in a single command line.
   Allow rekeying of even files whose content is not currently available.
   Use with caution.
 
+* `--batch`
+
+  Enables batch mode, in which lines are read from stdin.
+  Each line should contain the file, and the new key to use for that file,
+  separated by a single space.
+
 # SEE ALSO
 
 [[git-annex]](1)

From e9c44536b2f4bcb03c1cf265a0b8f333da155973 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 5 Dec 2016 13:52:40 -0400
Subject: [PATCH 173/367] fix formatting

mdwn2man gets confused as `command` spanning multiple lines..
---
 doc/git-annex-add.mdwn | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/git-annex-add.mdwn b/doc/git-annex-add.mdwn
index b65ed51325..15bb8a6a0c 100644
--- a/doc/git-annex-add.mdwn
+++ b/doc/git-annex-add.mdwn
@@ -15,9 +15,9 @@ If no path is specified, adds files from the current directory and below.
 Files that are already checked into git and are unmodified, or that
 git has been configured to ignore will be silently skipped.
 
-If annex.largefiles is configured, and does not match a file, `git annex
-add` will behave the same as `git add` and add the non-large file directly
-to the git repository, instead of to the annex.
+If annex.largefiles is configured, and does not match a file, 
+`git annex add` will behave the same as `git add` and add the
+non-large file directly to the git repository, instead of to the annex.
 
 Large files are added to the annex in locked form, which prevents further
 modification of their content unless unlocked by [[git-annex-unlock]](1).

From f61508aed47bb915529f35f2281407c0fc613dfe Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 5 Dec 2016 14:02:11 -0400
Subject: [PATCH 174/367] add: Stage modified non-large files when running in
 indirect mode.

(This was already done in v6 mode and direct mode.)
---
 CHANGELOG      |  2 ++
 Command/Add.hs | 10 +++-------
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index c30447f4e2..8a7c6cce64 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -14,6 +14,8 @@ git-annex (6.20161119) UNRELEASED; urgency=medium
   * fromkey: Accept multiple pairs of files and keys.
     Thanks, Daniel Brooks.
   * rekey: Added --batch mode.
+  * add: Stage modified non-large files when running in indirect mode. 
+    (This was already done in v6 mode and direct mode.)
 
  -- Joey Hess <id@joeyh.name>  Mon, 21 Nov 2016 11:27:50 -0400
 
diff --git a/Command/Add.hs b/Command/Add.hs
index eeaaf5d342..f9cfbb9a14 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -41,9 +41,6 @@ optParser desc = AddOptions
 		)
 	<*> parseBatchOption
 
-{- Add acts on both files not checked into git yet, and unlocked files.
- -
- - In direct mode, it acts on any files that have changed. -}
 seek :: AddOptions -> CommandSeek
 seek o = allowConcurrentOutput $ do
 	matcher <- largeFilesMatcher
@@ -59,10 +56,9 @@ seek o = allowConcurrentOutput $ do
 		NoBatch -> do
 			let go a = a gofile (addThese o)
 			go (withFilesNotInGit (not $ includeDotFiles o))
-			ifM (versionSupportsUnlockedPointers <||> isDirect)
-				( go withFilesMaybeModified
-				, go withFilesOldUnlocked
-				)
+			go withFilesMaybeModified
+			unlessM (versionSupportsUnlockedPointers <||> isDirect) $
+				go withFilesOldUnlocked
 
 {- Pass file off to git-add. -}
 startSmall :: FilePath -> CommandStart

From 3e0176054b34684cf7cf30e861bbe00b5aa2525d Mon Sep 17 00:00:00 2001
From: alpernebbi <alpernebbi@web>
Date: Mon, 5 Dec 2016 19:34:55 +0000
Subject: [PATCH 175/367]

---
 doc/tips/a_gui_for_metadata_operations.mdwn | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/tips/a_gui_for_metadata_operations.mdwn

diff --git a/doc/tips/a_gui_for_metadata_operations.mdwn b/doc/tips/a_gui_for_metadata_operations.mdwn
new file mode 100644
index 0000000000..1e11800682
--- /dev/null
+++ b/doc/tips/a_gui_for_metadata_operations.mdwn
@@ -0,0 +1,13 @@
+Hey everyone.
+
+I wrote a GUI for git-annex metadata in Python: [git-annex-metadata-gui](https://github.com/alpernebbi/git-annex-metadata-gui). 
+It shows the files that are in the current branch (only those in the annex) in the respective folder hierarchy.
+The keys that are in the repository, but not in the current branch are also shown in another tab.
+You can view, edit or remove fields for individual files with support for multiple values for fields.
+There is a file preview for image and text files as well.
+I uploaded some screenshots in the repository to show it in action.
+
+While making it, I decided to move the git-annex calls into its own Python package, 
+which became [git-annex-adapter](https://github.com/alpernebbi/git-annex-adapter).
+
+I hope these can be useful to someone other than myself as well.

From 0e1c2ff1a4aac0f20384736755297148dd91f8de Mon Sep 17 00:00:00 2001
From: alpernebbi <alpernebbi@web>
Date: Mon, 5 Dec 2016 20:46:07 +0000
Subject: [PATCH 176/367] Added a comment: UTF-8 problems in some other
 commands

---
 ..._aa6fe46ee76dd8bfa9a56cbd5131cb8b._comment | 55 +++++++++++++++++++
 1 file changed, 55 insertions(+)
 create mode 100644 doc/bugs/git-annex_fromkey_barfs_on_utf-8_input/comment_1_aa6fe46ee76dd8bfa9a56cbd5131cb8b._comment

diff --git a/doc/bugs/git-annex_fromkey_barfs_on_utf-8_input/comment_1_aa6fe46ee76dd8bfa9a56cbd5131cb8b._comment b/doc/bugs/git-annex_fromkey_barfs_on_utf-8_input/comment_1_aa6fe46ee76dd8bfa9a56cbd5131cb8b._comment
new file mode 100644
index 0000000000..a0409e2810
--- /dev/null
+++ b/doc/bugs/git-annex_fromkey_barfs_on_utf-8_input/comment_1_aa6fe46ee76dd8bfa9a56cbd5131cb8b._comment
@@ -0,0 +1,55 @@
+[[!comment format=mdwn
+ username="alpernebbi"
+ avatar="http://cdn.libravatar.org/avatar/daf2abb14f39e28ad75d5f9a03fcd106"
+ subject="UTF-8 problems in some other commands"
+ date="2016-12-05T20:46:07Z"
+ content="""
+Running the command above gives me the same error on Xubuntu 16.04, using `git-annex-standalone` package from NeuroDebian repositories.
+
+    git-annex version: 6.20161122+gitg9f179ae-1~ndall+1
+    build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+    remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+    local repository version: 5
+    supported repository versions: 3 5 6
+    upgrade supported from repository versions: 0 1 2 3 4 5
+    operating system: linux x86_64
+
+I encountered other commands that fail as well:
+
+    $ touch u.txt ü.txt
+    $ git annex add
+
+    $ git-annex calckey ü.txt
+    # prints key
+
+    $ git-annex calckey --batch
+    ü.txt
+    # dies
+
+    $ git-annex lookupkey ü.txt
+    # prints key
+
+    $ git-annex lookupkey --batch
+    ü.txt
+    # dies
+
+    $ git-annex metadata --batch --json
+    {\"file\":\"ü.txt\"}
+    # dies
+
+    $ git-annex metadata --batch --json
+    {\"file\":\"u.txt\",\"fields\":{\"ü\":[\"b\"]}}
+    # dies
+
+    $ git-annex metadata --batch --json
+    {\"file\":\"u.txt\",\"fields\":{\"a\":[\"ü\"]}}
+    # dies
+
+All those die without output, all $? are 0.
+No values were recorded to metadata.
+Also:
+
+    $ git-annex-metadata --json
+    # entry for \"ü.txt\" has \"file\":\"��.txt\"
+"""]]

From 7222ed2b206847d09ffbcdc5ea5731c2a656c3aa Mon Sep 17 00:00:00 2001
From: alpernebbi <alpernebbi@web>
Date: Mon, 5 Dec 2016 21:55:55 +0000
Subject: [PATCH 177/367]

---
 ...y_modified_in_the_same_batch_mode_run.mdwn | 84 +++++++++++++++++++
 1 file changed, 84 insertions(+)
 create mode 100644 doc/bugs/Metadata_values_get_stuck_when_repeatedly_modified_in_the_same_batch_mode_run.mdwn

diff --git a/doc/bugs/Metadata_values_get_stuck_when_repeatedly_modified_in_the_same_batch_mode_run.mdwn b/doc/bugs/Metadata_values_get_stuck_when_repeatedly_modified_in_the_same_batch_mode_run.mdwn
new file mode 100644
index 0000000000..1ec07dccbe
--- /dev/null
+++ b/doc/bugs/Metadata_values_get_stuck_when_repeatedly_modified_in_the_same_batch_mode_run.mdwn
@@ -0,0 +1,84 @@
+### Please describe the problem.
+
+While running `git-annex metadata --batch --json`, repeatedly assigning a field to the same value in the same run (with different values in between the assignments of the same value) causes a value to get stuck.
+
+### What steps will reproduce the problem?
+
+    $ touch test.txt
+    $ git annex add
+    $ git-annex metadata --batch --json
+    {"file":"test.txt","fields":{"f":["a"]}}
+    # prints { ... "f":["a"] ... }
+    {"file":"test.txt","fields":{"f":["b"]}}
+    # prints { ... "f":["b"] ... }
+    {"file":"test.txt","fields":{"f":["c"]}}
+    # prints { ... "f":["c"] ... }
+    {"file":"test.txt","fields":{"f":["a"]}}
+    # prints { ... "f":["a", "c"] ... }
+    {"file":"test.txt","fields":{"f":["b"]}}
+    # prints { ... "f":["c"] ... }
+    
+### What version of git-annex are you using? On what operating system?
+
+    git-annex version: 6.20161122+gitg9f179ae-1~ndall+1
+    build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+    remote types: git gcrypt S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+    local repository version: 5
+    supported repository versions: 3 5 6
+    upgrade supported from repository versions: 0 1 2 3 4 5
+    operating system: linux x86_64
+
+I'm using Xubuntu 16.04, with the `git-annex-standalone` package from NeuroDebian repository.
+
+### Please provide any additional information below.
+
+If you keep reassigning the same values, things get very weird. Full inputs/outputs from a sample run:
+
+    {"file":"test.txt","fields":{"f":["a"]}}
+    {"command":"metadata","note":"f=a\nf-lastchanged=2016-12-05@21-17-39\nlastchanged=2016-12-05@21-17-39\n","success":true,"key":"SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.txt","file":"test.txt","fields":    {"f-lastchanged":["2016-12-05@21-17-39"],"lastchanged":["2016-12-05@21-17-39"],"f":["a"]}}
+    {"file":"test.txt","fields":{"f":["b"]}}
+    {"command":"metadata","note":"f=b\nf-lastchanged=2016-12-05@21-17-39\nlastchanged=2016-12-05@21-17-39\n","success":true,"key":"SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.txt","file":"test.txt","fields":{"f-lastchanged":["2016-12-05@21-17-39"],"lastchanged":["2016-12-05@21-17-39"],"f":["b"]}}
+    {"file":"test.txt","fields":{"f":["c"]}}
+    {"command":"metadata","note":"f=c\nf-lastchanged=2016-12-05@21-17-39\nlastchanged=2016-12-05@21-17-39\n","success":true,"key":"SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.txt","file":"test.txt","fields":{"f-lastchanged":["2016-12-05@21-17-39"],"lastchanged":["2016-12-05@21-17-39"],"f":["c"]}}
+    {"file":"test.txt","fields":{"f":["a"]}}
+    {"command":"metadata","note":"f=a\nf=c\nf-lastchanged=2016-12-05@21-17-39\nlastchanged=2016-12-05@21-17-39\n","success":true,"key":"SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.txt","file":"test.txt","fields":{"f-lastchanged":["2016-12-05@21-17-39"],"lastchanged":["2016-12-05@21-17-39"],"f":["a","c"]}}
+    {"file":"test.txt","fields":{"f":["b"]}}
+    {"command":"metadata","note":"f=c\nf-lastchanged=2016-12-05@21-17-39\nlastchanged=2016-12-05@21-17-39\n","success":true,"key":"SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.txt","file":"test.txt","fields":{"f-lastchanged":["2016-12-05@21-17-39"],"lastchanged":["2016-12-05@21-17-39"],"f":["c"]}}
+    {"file":"test.txt","fields":{"f":[]}}   
+    {"command":"metadata","note":"lastchanged=2016-12-05@21-17-39\n","success":true,"key":"SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.txt","file":"test.txt","fields":{"lastchanged":["2016-12-05@21-17-39"]}}
+    {"file":"test.txt","fields":{"f":["b"]}}
+    {"command":"metadata","note":"lastchanged=2016-12-05@21-17-39\n","success":true,"key":"SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.txt","file":"test.txt","fields":{"lastchanged":["2016-12-05@21-17-39"]}}
+    {"file":"test.txt","fields":{"f":["c"]}}
+    {"command":"metadata","note":"f=c\nf-lastchanged=2016-12-05@21-17-39\nlastchanged=2016-12-05@21-17-39\n","success":true,"key":"SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.txt","file":"test.txt","fields":{"f-lastchanged":["2016-12-05@21-17-39"],"lastchanged":["2016-12-05@21-17-39"],"f":["c"]}}
+    {"file":"test.txt","fields":{"f":["a"]}}
+    {"command":"metadata","note":"f=a\nf=c\nf-lastchanged=2016-12-05@21-17-39\nlastchanged=2016-12-05@21-17-39\n","success":true,"key":"SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.txt","file":"test.txt","fields":{"f-lastchanged":["2016-12-05@21-17-39"],"lastchanged":["2016-12-05@21-17-39"],"f":["a","c"]}}
+    {"file":"test.txt","fields":{"f":["b"]}}
+    {"command":"metadata","note":"f=c\nf-lastchanged=2016-12-05@21-17-39\nlastchanged=2016-12-05@21-17-39\n","success":true,"key":"SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.txt","file":"test.txt","fields":{"f-lastchanged":["2016-12-05@21-17-39"],"lastchanged":["2016-12-05@21-17-39"],"f":["c"]}}
+    {"file":"test.txt","fields":{"f":["c"]}}
+    {"command":"metadata","note":"f=c\nf-lastchanged=2016-12-05@21-17-39\nlastchanged=2016-12-05@21-17-39\n","success":true,"key":"SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.txt","file":"test.txt","fields":{"f-lastchanged":["2016-12-05@21-17-39"],"lastchanged":["2016-12-05@21-17-39"],"f":["c"]}}
+    {"file":"test.txt","fields":{"f":["a"]}}
+    {"command":"metadata","note":"f=a\nf=c\nf-lastchanged=2016-12-05@21-17-39\nlastchanged=2016-12-05@21-17-39\n","success":true,"key":"SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.txt","file":"test.txt","fields":{"f-lastchanged":["2016-12-05@21-17-39"],"lastchanged":["2016-12-05@21-17-39"],"f":["a","c"]}}
+    {"file":"test.txt","fields":{"f":["b"]}}
+    {"command":"metadata","note":"f=c\nf-lastchanged=2016-12-05@21-17-39\nlastchanged=2016-12-05@21-17-39\n","success":true,"key":"SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.txt","file":"test.txt","fields":{"f-lastchanged":["2016-12-05@21-17-39"],"lastchanged":["2016-12-05@21-17-39"],"f":["c"]}}
+    {"file":"test.txt","fields":{"f":["b"]}}
+    {"command":"metadata","note":"f=b\nf=c\nf-lastchanged=2016-12-05@21-17-39\nlastchanged=2016-12-05@21-17-39\n","success":true,"key":"SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.txt","file":"test.txt","fields":{"f-lastchanged":["2016-12-05@21-17-39"],"lastchanged":["2016-12-05@21-17-39"],"f":["b","c"]}}
+    {"file":"test.txt","fields":{"f":["a"]}}
+    {"command":"metadata","note":"f=a\nf=b\nf=c\nf-lastchanged=2016-12-05@21-17-39\nlastchanged=2016-12-05@21-17-39\n","success":true,"key":"SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.txt","file":"test.txt","fields":{"f-lastchanged":["2016-12-05@21-17-39"],"lastchanged":["2016-12-05@21-17-39"],"f":["a","b","c"]}}
+    {"file":"test.txt","fields":{"f":["d"]}}
+    {"command":"metadata","note":"f=b\nf=c\nf=d\nf-lastchanged=2016-12-05@21-17-39\nlastchanged=2016-12-05@21-17-39\n","success":true,"key":"SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.txt","file":"test.txt","fields":{"f-lastchanged":["2016-12-05@21-17-39"],"lastchanged":["2016-12-05@21-17-39"],"f":["b","c","d"]}}
+    {"file":"test.txt","fields":{"f":["a"]}}
+    {"command":"metadata","note":"f=b\nf=c\nf=d\nf-lastchanged=2016-12-05@21-17-39\nlastchanged=2016-12-05@21-17-39\n","success":true,"key":"SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.txt","file":"test.txt","fields":{"f-lastchanged":["2016-12-05@21-17-39"],"lastchanged":["2016-12-05@21-17-39"],"f":["b","c","d"]}}
+    {"file":"test.txt","fields":{"f":["a"]}}
+    {"command":"metadata","note":"f=b\nf=c\nf=d\nf-lastchanged=2016-12-05@21-17-39\nlastchanged=2016-12-05@21-17-39\n","success":true,"key":"SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.txt","file":"test.txt","fields":{"f-lastchanged":["2016-12-05@21-17-39"],"lastchanged":["2016-12-05@21-17-39"],"f":["b","c","d"]}}
+    {"file":"test.txt","fields":{"f":[]}}   
+    {"command":"metadata","note":"f=c\nf-lastchanged=2016-12-05@21-17-39\nlastchanged=2016-12-05@21-17-39\n","success":true,"key":"SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.txt","file":"test.txt","fields":{"f-lastchanged":["2016-12-05@21-17-39"],"lastchanged":["2016-12-05@21-17-39"],"f":["c"]}}
+    
+Restarting the process solves the issue.
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+I love the metadata functionality so much that I wrote [[tips/a_gui_for_metadata_operations]] and discovered this bug.
+Metadata driven views are awesome (but I don't like the entire folder hierarchy being appended to the filename).
+I haven't used the other commands much since I have not yet organized most of my stuff (and their naively copy-pasted backups), but I am glad I discovered git-annex before I began organizing.
+

From b8e27383fded9489ae065600edd9e2a3edcaa6d6 Mon Sep 17 00:00:00 2001
From: "0xloem@0bd8a79a57e4f0dcade8fc81d162c37eae4d6730" <0xloem@web>
Date: Tue, 6 Dec 2016 13:01:51 +0000
Subject: [PATCH 178/367]

---
 ...ecommit_memory__58___Invalid_argument.mdwn | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)
 create mode 100644 doc/bugs/unable_to_decommit_memory__58___Invalid_argument.mdwn

diff --git a/doc/bugs/unable_to_decommit_memory__58___Invalid_argument.mdwn b/doc/bugs/unable_to_decommit_memory__58___Invalid_argument.mdwn
new file mode 100644
index 0000000000..44a6d2d12e
--- /dev/null
+++ b/doc/bugs/unable_to_decommit_memory__58___Invalid_argument.mdwn
@@ -0,0 +1,25 @@
+### Please describe the problem.
+Recent builds of git-annex spew out many lines such as:
+```
+git-annex: unable to decommit memory: Invalid argument
+git-annex: unable to decommit memory: Invalid argument
+git-annex: unable to decommit memory: Invalid argument
+git-annex: unable to decommit memory: Invalid argument
+git-annex: unable to decommit memory: Invalid argument
+```
+
+### What steps will reproduce the problem?
+This happens to me syncing any large repository now.
+
+### What version of git-annex are you using? On what operating system?
+git-annex version: 6.20161118-g0a34f08
+uname -r: 4.4.14-11.pvops.qubes.x86_64
+/etc/system-release: Fedora release 23 (Twenty Three)
+
+### Please provide any additional information below.
+
+I found this: https://ghc.haskell.org/trac/ghc/ticket/12495
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Git annex rocks!

From 89ff5bece0da434c8c3f837670e022163f2e45b0 Mon Sep 17 00:00:00 2001
From: "0xloem@0bd8a79a57e4f0dcade8fc81d162c37eae4d6730" <0xloem@web>
Date: Tue, 6 Dec 2016 13:03:51 +0000
Subject: [PATCH 179/367]

---
 ...to_decommit_memory__58___Invalid_argument.mdwn | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/doc/bugs/unable_to_decommit_memory__58___Invalid_argument.mdwn b/doc/bugs/unable_to_decommit_memory__58___Invalid_argument.mdwn
index 44a6d2d12e..5d8e55bec1 100644
--- a/doc/bugs/unable_to_decommit_memory__58___Invalid_argument.mdwn
+++ b/doc/bugs/unable_to_decommit_memory__58___Invalid_argument.mdwn
@@ -1,19 +1,20 @@
 ### Please describe the problem.
 Recent builds of git-annex spew out many lines such as:
-```
-git-annex: unable to decommit memory: Invalid argument
-git-annex: unable to decommit memory: Invalid argument
-git-annex: unable to decommit memory: Invalid argument
-git-annex: unable to decommit memory: Invalid argument
-git-annex: unable to decommit memory: Invalid argument
-```
+
+    git-annex: unable to decommit memory: Invalid argument
+    git-annex: unable to decommit memory: Invalid argument
+    git-annex: unable to decommit memory: Invalid argument
+    git-annex: unable to decommit memory: Invalid argument
+    git-annex: unable to decommit memory: Invalid argument
 
 ### What steps will reproduce the problem?
 This happens to me syncing any large repository now.
 
 ### What version of git-annex are you using? On what operating system?
 git-annex version: 6.20161118-g0a34f08
+
 uname -r: 4.4.14-11.pvops.qubes.x86_64
+
 /etc/system-release: Fedora release 23 (Twenty Three)
 
 ### Please provide any additional information below.

From 3dc64fe86774e29ee6cf0215f63b6785f9ac3a6c Mon Sep 17 00:00:00 2001
From: "0xloem@0bd8a79a57e4f0dcade8fc81d162c37eae4d6730" <0xloem@web>
Date: Tue, 6 Dec 2016 13:56:13 +0000
Subject: [PATCH 180/367]

---
 doc/bugs/unable_to_decommit_memory__58___Invalid_argument.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/unable_to_decommit_memory__58___Invalid_argument.mdwn b/doc/bugs/unable_to_decommit_memory__58___Invalid_argument.mdwn
index 5d8e55bec1..572e0406a3 100644
--- a/doc/bugs/unable_to_decommit_memory__58___Invalid_argument.mdwn
+++ b/doc/bugs/unable_to_decommit_memory__58___Invalid_argument.mdwn
@@ -21,6 +21,8 @@ uname -r: 4.4.14-11.pvops.qubes.x86_64
 
 I found this: https://ghc.haskell.org/trac/ghc/ticket/12495
 
+It looks like this is a problem that occurs only on kernels < 4.5, when ghc is built with a newer glibc, I think.
+
 ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
 
 Git annex rocks!

From 7a91c4952bd3778ee2a58fe1ce50ee76fdd8ea75 Mon Sep 17 00:00:00 2001
From: "0xloem@0bd8a79a57e4f0dcade8fc81d162c37eae4d6730" <0xloem@web>
Date: Tue, 6 Dec 2016 14:59:40 +0000
Subject: [PATCH 181/367]

---
 doc/install/fromsource.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/install/fromsource.mdwn b/doc/install/fromsource.mdwn
index c46321099b..7973e3dc93 100644
--- a/doc/install/fromsource.mdwn
+++ b/doc/install/fromsource.mdwn
@@ -83,7 +83,7 @@ Get the git-annex source code, and inside the source tree, run:
 
 To build with all features enabled, including the assistant and webapp,
 you will need to install several C libraries and their headers,
-including libgnutls, libgsasl, libxml2, libmagic, and zlib. How to do
+including libgnutls, libgsasl, libxml2, libmagic, zlib, and chrpath. How to do
 that for your OS is beyond the scope of this page. 
 
 Once the C libraries are installed, run inside the source tree:

From b29088b8dc9fe543248bb29f03b096f85eb92019 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 6 Dec 2016 12:19:47 -0400
Subject: [PATCH 182/367] stub Remote.P2P

Similar to GCrypt remotes, P2P remotes have an url, so Remote.Git has to
separate them out and handle them, passing off to Remote.P2P.

This commit was sponsored by Ignacio on Patreon.
---
 P2P/Address.hs  |  5 +++
 Remote/Git.hs   |  6 +++-
 Remote/List.hs  |  4 ++-
 Remote/P2P.hs   | 85 +++++++++++++++++++++++++++++++++++++++++++++++++
 git-annex.cabal |  1 +
 5 files changed, 99 insertions(+), 2 deletions(-)
 create mode 100644 Remote/P2P.hs

diff --git a/P2P/Address.hs b/P2P/Address.hs
index 19ff82a89b..09ffc79735 100644
--- a/P2P/Address.hs
+++ b/P2P/Address.hs
@@ -10,6 +10,7 @@ module P2P.Address where
 import qualified Annex
 import Annex.Common
 import Git
+import Git.Types
 import Creds
 import Utility.AuthToken
 import Utility.Tor
@@ -54,6 +55,10 @@ instance FormatP2PAddress P2PAddressAuth where
 		authtoken <- toAuthToken (T.pack $ reverse ra)
 		return (P2PAddressAuth addr authtoken)
 
+repoP2PAddress :: Repo -> Maybe P2PAddress
+repoP2PAddress (Repo { location = Url url }) = unformatP2PAddress (show url)
+repoP2PAddress _ = Nothing
+
 -- | Load known P2P addresses for this repository.
 loadP2PAddresses :: Annex [P2PAddress]
 loadP2PAddresses = mapMaybe unformatP2PAddress . maybe [] lines
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 3304e2069f..41fb46e82e 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -49,6 +49,8 @@ import Remote.Helper.Git
 import Remote.Helper.Messages
 import qualified Remote.Helper.Ssh as Ssh
 import qualified Remote.GCrypt
+import qualified Remote.P2P
+import P2P.Address
 import Annex.Path
 import Creds
 import Annex.CatFile
@@ -130,7 +132,9 @@ configRead autoinit r = do
 gen :: Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remote)
 gen r u c gc
 	| Git.GCrypt.isEncrypted r = Remote.GCrypt.chainGen r u c gc
-	| otherwise = go <$> remoteCost gc defcst
+	| otherwise = case repoP2PAddress r of
+		Nothing -> go <$> remoteCost gc defcst
+		Just addr -> Remote.P2P.chainGen addr r u c gc
   where
 	defcst = if repoCheap r then cheapRemoteCost else expensiveRemoteCost
 	go cst = Just new
diff --git a/Remote/List.hs b/Remote/List.hs
index 9c231b124f..a5e305622f 100644
--- a/Remote/List.hs
+++ b/Remote/List.hs
@@ -23,6 +23,7 @@ import qualified Git.Config
 
 import qualified Remote.Git
 import qualified Remote.GCrypt
+import qualified Remote.P2P
 #ifdef WITH_S3
 import qualified Remote.S3
 #endif
@@ -44,6 +45,7 @@ remoteTypes :: [RemoteType]
 remoteTypes =
 	[ Remote.Git.remote
 	, Remote.GCrypt.remote
+	, Remote.P2P.remote
 #ifdef WITH_S3
 	, Remote.S3.remote
 #endif
@@ -116,4 +118,4 @@ updateRemote remote = do
 {- Checks if a remote is syncable using git. -}
 gitSyncableRemote :: Remote -> Bool
 gitSyncableRemote r = remotetype r `elem`
-	[ Remote.Git.remote, Remote.GCrypt.remote ]
+	[ Remote.Git.remote, Remote.GCrypt.remote, Remote.P2P.remote ]
diff --git a/Remote/P2P.hs b/Remote/P2P.hs
new file mode 100644
index 0000000000..e0428eeeb5
--- /dev/null
+++ b/Remote/P2P.hs
@@ -0,0 +1,85 @@
+{- git remotes using the git-annex P2P protocol
+ -
+ - Copyright 2016 Joey Hess <id@joeyh.name>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Remote.P2P (
+	remote,
+	chainGen
+) where
+
+import Annex.Common
+import P2P.Address
+import Types.Remote
+import Types.GitConfig
+import qualified Git
+import Config
+import Config.Cost
+import Remote.Helper.Git
+import Remote.Helper.Special
+
+remote :: RemoteType
+remote = RemoteType {
+	typename = "p2p",
+	-- Remote.Git takes care of enumerating P2P remotes,
+	-- and will call chainGen on them.
+	enumerate = const (return []),
+	generate = \_ _ _ _ -> return Nothing,
+	setup = error "P2P remotes are set up using git-annex p2p"
+}
+
+chainGen :: P2PAddress -> Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remote)
+chainGen addr r u c gc = do
+	workerpool <- mkWorkerPool addr
+	cst <- remoteCost gc expensiveRemoteCost
+	let this = Remote 
+		{ uuid = u
+		, cost = cst
+		, name = Git.repoDescribe r
+		, storeKey = storeKeyDummy
+		, retrieveKeyFile = retreiveKeyFileDummy
+		, retrieveKeyFileCheap = \_ _ _ -> return False
+		, removeKey = removeKeyDummy
+		, lockContent = Nothing -- TODO use p2p protocol locking
+		, checkPresent = checkPresentDummy
+		, checkPresentCheap = False
+		, whereisKey = Nothing
+		, remoteFsck = Nothing
+		, repairRepo = Nothing
+		, config = c
+		, localpath = Nothing
+		, repo = r
+		, gitconfig = gc { remoteGitConfig = Just $ extractGitConfig r }
+		, readonly = False
+		, availability = GloballyAvailable
+		, remotetype = remote
+		, mkUnavailable = return Nothing
+		, getInfo = gitRepoInfo this
+		, claimUrl = Nothing
+		, checkUrl = Nothing
+	}
+	return $ Just $ specialRemote' (specialRemoteCfg c) c
+		(simplyPrepare $ store this workerpool)
+		(simplyPrepare $ retrieve this workerpool)
+		(simplyPrepare $ remove this workerpool)
+		(simplyPrepare $ checkKey this workerpool)
+		this
+
+data WorkerPool = WorkerPool
+
+mkWorkerPool :: P2PAddress -> Annex WorkerPool
+mkWorkerPool addr = undefined
+
+store :: Remote -> WorkerPool -> Storer
+store r workerpool = undefined
+
+retrieve :: Remote -> WorkerPool -> Retriever
+retrieve r workerpool = undefined
+
+remove :: Remote -> WorkerPool -> Remover
+remove r workerpool k = undefined
+
+checkKey :: Remote -> WorkerPool -> CheckPresent
+checkKey r workerpool k = undefined
diff --git a/git-annex.cabal b/git-annex.cabal
index f6d8c54823..7fcba0623c 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -937,6 +937,7 @@ Executable git-annex
     Remote.Helper.Tor
     Remote.Hook
     Remote.List
+    Remote.P2P
     Remote.Rsync
     Remote.Rsync.RsyncUrl
     Remote.S3

From 2bd2e0880cc09ccb77cfdd10c0c414bad85f50b0 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 6 Dec 2016 15:05:44 -0400
Subject: [PATCH 183/367] added StoreContentTo

This is needed in addition to StoreContent, because retrieveKeyFile can
be used to retrieve to different destination files, not only the tmp
file for a key.

This commit was sponsored by Ole-Morten Duesund on Patreon.
---
 P2P/Annex.hs    | 32 ++++++++++++++++++++------------
 P2P/Protocol.hs | 44 +++++++++++++++++++++++++++++---------------
 2 files changed, 49 insertions(+), 27 deletions(-)

diff --git a/P2P/Annex.hs b/P2P/Annex.hs
index 5e1763fc6e..d0c00def37 100644
--- a/P2P/Annex.hs
+++ b/P2P/Annex.hs
@@ -16,7 +16,6 @@ module P2P.Annex
 import Annex.Common
 import Annex.Content
 import Annex.Transfer
-import Annex.Notification
 import P2P.Protocol
 import P2P.IO
 import Logs.Location
@@ -46,6 +45,9 @@ runLocal runmode runner a = case a of
 		tmp <- fromRepo $ gitAnnexTmpObjectLocation k
 		size <- liftIO $ catchDefaultIO 0 $ getFileSize tmp
 		runner (next (Len size))
+	FileSize f next -> do
+		size <- liftIO $ catchDefaultIO 0 $ getFileSize f
+		runner (next (Len size))
 	ContentSize k next -> do
 		let getsize = liftIO . catchMaybeIO . getFileSize
 		size <- inAnnex' isJust Nothing getsize k
@@ -69,16 +71,15 @@ runLocal runmode runner a = case a of
 					Left _ -> return Nothing
 					Right b -> runner (next b)
 			_ -> return Nothing
-	WriteContent k af (Offset o) (Len l) b next -> do
+	StoreContent k af o l b next -> do
 		ok <- flip catchNonAsync (const $ return False) $
 			transfer download k af $
-				getViaTmp AlwaysVerify k $ \tmp -> liftIO $ do
-					withBinaryFile tmp WriteMode $ \h -> do
-						when (o /= 0) $
-							hSeek h AbsoluteSeek o
-						L.hPut h b
-					sz <- getFileSize tmp
-					return (toInteger sz == l, UnVerified)
+				getViaTmp AlwaysVerify k $ \tmp ->
+					unVerified $ storefile tmp o l b
+		runner (next ok)
+	StoreContentTo dest o l b next -> do
+		ok <- flip catchNonAsync (const $ return False) $
+			storefile dest o l b
 		runner (next ok)
 	SetPresent k u next -> do
 		v <- tryNonAsync $ logChange k u InfoPresent
@@ -111,10 +112,17 @@ runLocal runmode runner a = case a of
 				next
 			Right _ -> runner next
   where
-	transfer mk k af a = case runmode of
+	transfer mk k af ta = case runmode of
 		-- Update transfer logs when serving.
 		Serving theiruuid -> 
-			mk theiruuid k af noRetry (const a) noNotification
+			mk theiruuid k af noRetry (const ta) noNotification
 		-- Transfer logs are updated higher in the stack when
 		-- a client.
-		Client -> a
+		Client -> ta
+	storefile dest (Offset o) (Len l) b = liftIO $ do
+		withBinaryFile dest WriteMode $ \h -> do
+			when (o /= 0) $
+				hSeek h AbsoluteSeek o
+			L.hPut h b
+		sz <- getFileSize dest
+		return (toInteger sz == l)
diff --git a/P2P/Protocol.hs b/P2P/Protocol.hs
index 53c3265efe..b2b734e48a 100644
--- a/P2P/Protocol.hs
+++ b/P2P/Protocol.hs
@@ -189,16 +189,25 @@ data LocalF c
 	= TmpContentSize Key (Len -> c)
 	-- ^ Gets size of the temp file where received content may have
 	-- been stored. If not present, returns 0.
+	| FileSize FilePath (Len -> c)
+	-- ^ Gets size of the content of a file. If not present, returns 0.
 	| ContentSize Key (Maybe Len -> c)
 	-- ^ Gets size of the content of a key, when the full content is
 	-- present.
 	| ReadContent Key AssociatedFile Offset (L.ByteString -> c)
 	-- ^ Lazily reads the content of a key. Note that the content
 	-- may change while it's being sent.
-	| WriteContent Key AssociatedFile Offset Len L.ByteString (Bool -> c)
-	-- ^ Writes content to temp file starting at an offset.
+	| StoreContent Key AssociatedFile Offset Len L.ByteString (Bool -> c)
+	-- ^ Stores content to the key's temp file starting at an offset.
 	-- Once the whole content of the key has been stored, moves the
-	-- temp file into place and returns True.
+	-- temp file into place as the content of the key, and returns True.
+	--
+	-- Note: The ByteString may not contain the entire remaining content
+	-- of the key. Only once the temp file size == Len has the whole
+	-- content been transferred.
+	| StoreContentTo FilePath Offset Len L.ByteString (Bool -> c)
+	-- ^ Stores the content to a temp file starting at an offset.
+	-- Once the whole content of the key has been stored, returns True.
 	--
 	-- Note: The ByteString may not contain the entire remaining content
 	-- of the key. Only once the temp file size == Len has the whole
@@ -246,16 +255,16 @@ checkPresent key = do
  -}
 lockContentWhile 
 	:: MonadMask m 
-	=> (forall r. Proto r -> m r)
+	=> (forall r. r -> Proto r -> m r)
 	-> Key
-	-> (Bool -> m ())
-	-> m ()
+	-> (Bool -> m a)
+	-> m a
 lockContentWhile runproto key a = bracket setup cleanup a
   where
-	setup = runproto $ do
+	setup = runproto False $ do
 		net $ sendMessage (LOCKCONTENT key)
 		checkSuccess
-	cleanup True = runproto $ net $ sendMessage UNLOCKCONTENT
+	cleanup True = runproto () $ net $ sendMessage UNLOCKCONTENT
 	cleanup False = return ()
 
 remove :: Key -> Proto Bool
@@ -263,8 +272,11 @@ remove key = do
 	net $ sendMessage (REMOVE key)
 	checkSuccess
 
-get :: Key -> AssociatedFile -> Proto Bool
-get key af = receiveContent key af (\offset -> GET offset af key)
+get :: FilePath -> Key -> AssociatedFile -> Proto Bool
+get dest key af = receiveContent sizer storer (\offset -> GET offset af key)
+  where
+	sizer = fileSize dest
+	storer = storeContentTo dest
 
 put :: Key -> AssociatedFile -> Proto Bool
 put key af = do
@@ -349,7 +361,9 @@ serveAuthed myuuid = void $ serverLoop handler
 		if have
 			then net $ sendMessage ALREADY_HAVE
 			else do
-				ok <- receiveContent key af PUT_FROM
+				let sizer = tmpContentSize key
+				let storer = storeContent key af
+				ok <- receiveContent sizer storer PUT_FROM
 				when ok $
 					local $ setPresent key myuuid
 		return ServerContinue
@@ -370,15 +384,15 @@ sendContent key af offset = do
 	net $ sendBytes len content
 	checkSuccess
 
-receiveContent :: Key -> AssociatedFile -> (Offset -> Message) -> Proto Bool
-receiveContent key af mkmsg = do
-	Len n <- local $ tmpContentSize key
+receiveContent :: Local Len -> (Offset -> Len -> L.ByteString -> Local Bool) -> (Offset -> Message) -> Proto Bool
+receiveContent sizer storer mkmsg = do
+	Len n <- local sizer
 	let offset = Offset n
 	net $ sendMessage (mkmsg offset)
 	r <- net receiveMessage
 	case r of
 		DATA len -> do
-			ok <- local . writeContent key af offset len
+			ok <- local . storer offset len
 				=<< net (receiveBytes len)
 			sendSuccess ok
 			return ok

From 26a53fb4a5244183a319614593ddb230de9acc8b Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 6 Dec 2016 15:08:00 -0400
Subject: [PATCH 184/367] finish implementation of Remote.P2P (untested)

Not tested at all, but it just might work.
Only known problem is that progress is not updated when storing to a P2P
remote.

This commit was sponsored by Nick Daly on Patreon.
---
 Remote/P2P.hs | 140 +++++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 116 insertions(+), 24 deletions(-)

diff --git a/Remote/P2P.hs b/Remote/P2P.hs
index e0428eeeb5..f97d76e71c 100644
--- a/Remote/P2P.hs
+++ b/Remote/P2P.hs
@@ -11,14 +11,23 @@ module Remote.P2P (
 ) where
 
 import Annex.Common
+import qualified Annex
+import qualified P2P.Protocol as P2P
 import P2P.Address
+import P2P.Annex
 import Types.Remote
 import Types.GitConfig
 import qualified Git
 import Config
 import Config.Cost
 import Remote.Helper.Git
-import Remote.Helper.Special
+import Remote.Helper.Tor
+import Utility.Tor
+import Utility.Metered
+import Types.NumCopies
+
+import Control.Concurrent
+import Control.Concurrent.STM
 
 remote :: RemoteType
 remote = RemoteType {
@@ -32,18 +41,18 @@ remote = RemoteType {
 
 chainGen :: P2PAddress -> Git.Repo -> UUID -> RemoteConfig -> RemoteGitConfig -> Annex (Maybe Remote)
 chainGen addr r u c gc = do
-	workerpool <- mkWorkerPool addr
+	connpool <- mkConnectionPool
 	cst <- remoteCost gc expensiveRemoteCost
 	let this = Remote 
 		{ uuid = u
 		, cost = cst
 		, name = Git.repoDescribe r
-		, storeKey = storeKeyDummy
-		, retrieveKeyFile = retreiveKeyFileDummy
+		, storeKey = store addr connpool
+		, retrieveKeyFile = retrieve addr connpool
 		, retrieveKeyFileCheap = \_ _ _ -> return False
-		, removeKey = removeKeyDummy
-		, lockContent = Nothing -- TODO use p2p protocol locking
-		, checkPresent = checkPresentDummy
+		, removeKey = remove addr connpool
+		, lockContent = Just (lock u addr connpool)
+		, checkPresent = checkpresent addr connpool
 		, checkPresentCheap = False
 		, whereisKey = Nothing
 		, remoteFsck = Nothing
@@ -60,26 +69,109 @@ chainGen addr r u c gc = do
 		, claimUrl = Nothing
 		, checkUrl = Nothing
 	}
-	return $ Just $ specialRemote' (specialRemoteCfg c) c
-		(simplyPrepare $ store this workerpool)
-		(simplyPrepare $ retrieve this workerpool)
-		(simplyPrepare $ remove this workerpool)
-		(simplyPrepare $ checkKey this workerpool)
-		this
+	return (Just this)
 
-data WorkerPool = WorkerPool
+-- TODO update progress
+store :: P2PAddress -> ConnectionPool -> Key -> AssociatedFile -> MeterUpdate -> Annex Bool
+store addr connpool k af p = fromMaybe False
+	<$> runProto addr connpool (P2P.put k af)
 
-mkWorkerPool :: P2PAddress -> Annex WorkerPool
-mkWorkerPool addr = undefined
+retrieve :: P2PAddress -> ConnectionPool -> Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex (Bool, Verification)
+retrieve addr connpool k af dest _p = unVerified $ fromMaybe False 
+	<$> runProto addr connpool (P2P.get dest k af)
 
-store :: Remote -> WorkerPool -> Storer
-store r workerpool = undefined
+remove :: P2PAddress -> ConnectionPool -> Key -> Annex Bool
+remove addr connpool k = fromMaybe False
+	<$> runProto addr connpool (P2P.remove k)
 
-retrieve :: Remote -> WorkerPool -> Retriever
-retrieve r workerpool = undefined
+checkpresent :: P2PAddress -> ConnectionPool -> Key -> Annex Bool
+checkpresent addr connpool k = maybe unavail return
+	=<< runProto addr connpool (P2P.checkPresent k)
+  where
+	unavail = giveup "can't connect to peer"
 
-remove :: Remote -> WorkerPool -> Remover
-remove r workerpool k = undefined
+lock :: UUID -> P2PAddress -> ConnectionPool -> Key -> (VerifiedCopy -> Annex r) -> Annex r
+lock theiruuid addr connpool k callback =
+	withConnection addr connpool $ \conn -> do
+		connv <- liftIO $ newMVar conn
+		let runproto d p = do
+			c <- liftIO $ takeMVar connv
+			(c', mr) <- runProto' p c
+			liftIO $ putMVar connv c'
+			return (fromMaybe d mr)
+		r <- P2P.lockContentWhile runproto k go
+		conn' <- liftIO $ takeMVar connv
+		return (conn', r)
+  where
+	go False = giveup "can't lock content"
+	go True = withVerifiedCopy LockedCopy theiruuid (return True) callback
 
-checkKey :: Remote -> WorkerPool -> CheckPresent
-checkKey r workerpool k = undefined
+-- | A connection to the peer.
+data Connection 
+	= TorAnnexConnection RunEnv
+	| ClosedConnection
+
+type ConnectionPool = TVar [Connection]
+
+mkConnectionPool :: Annex ConnectionPool
+mkConnectionPool = liftIO $ newTVarIO []
+
+-- Runs the Proto action.
+runProto :: P2PAddress -> ConnectionPool -> P2P.Proto a -> Annex (Maybe a)
+runProto addr connpool a = withConnection addr connpool (runProto' a)
+
+runProto' :: P2P.Proto a -> Connection -> Annex (Connection, Maybe a)
+runProto' _ ClosedConnection = return (ClosedConnection, Nothing)
+runProto' a conn@(TorAnnexConnection runenv) = do
+	r <- runFullProto Client runenv a
+	-- When runFullProto fails, the connection is no longer usable,
+	-- so close it.
+	if isJust r
+		then return (conn, r)
+		else do
+			liftIO $ hClose (runIhdl runenv)
+			return (ClosedConnection, r)
+
+-- Uses an open connection if one is available in the ConnectionPool;
+-- otherwise opens a new connection.
+--
+-- Once the action is done, the connection is added back to the
+-- ConnectionPool, unless it's no longer open.
+withConnection :: P2PAddress -> ConnectionPool -> (Connection -> Annex (Connection, a)) -> Annex a
+withConnection addr connpool a = bracketOnError get cache go
+  where
+	get = do
+		mc <- liftIO $ atomically $ do
+			l <- readTVar connpool
+			case l of
+				[] -> do
+					writeTVar connpool []
+					return Nothing
+				(c:cs) -> do
+					writeTVar connpool cs
+					return (Just c)
+		maybe (openConnection addr) return mc
+	
+	cache ClosedConnection = return ()
+	cache conn = liftIO $ atomically $ modifyTVar' connpool (conn:)
+
+	go conn = do
+		(conn', r) <- a conn
+		cache conn'
+		return r
+
+openConnection :: P2PAddress -> Annex Connection
+openConnection (TorAnnex onionaddress onionport) = do
+	v <- liftIO $ tryNonAsync $ 
+		torHandle =<< connectHiddenService onionaddress onionport
+	case v of
+		Right h -> do
+			g <- Annex.gitRepo
+			let runenv = RunEnv
+				{ runRepo = g
+				, runCheckAuth = const False
+				, runIhdl = h
+				, runOhdl = h
+				}
+			return (TorAnnexConnection runenv)
+		Left _e -> return ClosedConnection

From f744bd53912c429b49f3a7536fb5f6fd8a50e2f4 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 6 Dec 2016 15:40:31 -0400
Subject: [PATCH 185/367] refactor

---
 CmdLine/GitRemoteTorAnnex.hs  | 11 +-----
 P2P/Annex.hs                  |  8 ++--
 P2P/IO.hs                     | 73 +++++++++++++++++++++++------------
 Remote/Helper/Tor.hs          | 20 ----------
 Remote/P2P.hs                 | 30 ++++++--------
 RemoteDaemon/Transport/Tor.hs | 17 ++++----
 git-annex.cabal               |  1 -
 7 files changed, 74 insertions(+), 86 deletions(-)
 delete mode 100644 Remote/Helper/Tor.hs

diff --git a/CmdLine/GitRemoteTorAnnex.hs b/CmdLine/GitRemoteTorAnnex.hs
index 517ce7c82e..c4bf26c859 100644
--- a/CmdLine/GitRemoteTorAnnex.hs
+++ b/CmdLine/GitRemoteTorAnnex.hs
@@ -12,7 +12,6 @@ import qualified Annex
 import qualified Git.CurrentRepo
 import P2P.Protocol
 import P2P.IO
-import Remote.Helper.Tor
 import Utility.Tor
 import Utility.AuthToken
 import Annex.UUID
@@ -59,14 +58,8 @@ connectService address port service = do
 			<$> loadP2PRemoteAuthToken (TorAnnex address port)
 		myuuid <- getUUID
 		g <- Annex.gitRepo
-		h <- liftIO $ torHandle =<< connectHiddenService address port
-		let runenv = RunEnv
-			{ runRepo = g
-			, runCheckAuth = const False
-			, runIhdl = h
-			, runOhdl = h
-			}
-		liftIO $ runNetProto runenv $ do
+		conn <- liftIO $ connectPeer g (TorAnnex address port)
+		liftIO $ runNetProto conn $ do
 			v <- auth myuuid authtoken
 			case v of
 				Just _theiruuid -> connect service stdin stdout
diff --git a/P2P/Annex.hs b/P2P/Annex.hs
index d0c00def37..4105abe32e 100644
--- a/P2P/Annex.hs
+++ b/P2P/Annex.hs
@@ -9,7 +9,7 @@
 
 module P2P.Annex
 	( RunMode(..)
-	, RunEnv(..)
+	, P2PConnection(..)
 	, runFullProto
 	) where
 
@@ -31,12 +31,12 @@ data RunMode
 	| Client
 
 -- Full interpreter for Proto, that can receive and send objects.
-runFullProto :: RunMode -> RunEnv -> Proto a -> Annex (Maybe a)
-runFullProto runmode runenv = go
+runFullProto :: RunMode -> P2PConnection -> Proto a -> Annex (Maybe a)
+runFullProto runmode conn = go
   where
 	go :: RunProto Annex
 	go (Pure v) = pure (Just v)
-	go (Free (Net n)) = runNet runenv go n
+	go (Free (Net n)) = runNet conn go n
 	go (Free (Local l)) = runLocal runmode go l
 
 runLocal :: RunMode -> RunProto Annex -> LocalF (Proto a) -> Annex (Maybe a)
diff --git a/P2P/IO.hs b/P2P/IO.hs
index fb621ab2b1..f63b2808b3 100644
--- a/P2P/IO.hs
+++ b/P2P/IO.hs
@@ -9,12 +9,15 @@
 
 module P2P.IO
 	( RunProto
-	, RunEnv(..)
+	, P2PConnection(..)
+	, connectPeer
+	, setupHandle
 	, runNetProto
 	, runNet
 	) where
 
 import P2P.Protocol
+import P2P.Address
 import Utility.Process
 import Git
 import Git.Command
@@ -22,11 +25,14 @@ import Utility.AuthToken
 import Utility.SafeCommand
 import Utility.SimpleProtocol
 import Utility.Exception
+import Utility.Tor
+import Utility.FileSystemEncoding
 
 import Control.Monad
 import Control.Monad.Free
 import Control.Monad.IO.Class
 import System.Exit (ExitCode(..))
+import Network.Socket
 import System.IO
 import Control.Concurrent
 import Control.Concurrent.Async
@@ -36,41 +42,60 @@ import qualified Data.ByteString.Lazy as L
 -- Type of interpreters of the Proto free monad.
 type RunProto m = forall a. (MonadIO m, MonadMask m) => Proto a -> m (Maybe a)
 
-data RunEnv = RunEnv
-	{ runRepo :: Repo
-	, runCheckAuth :: (AuthToken -> Bool)
-	, runIhdl :: Handle
-	, runOhdl :: Handle
+data P2PConnection = P2PConnection
+	{ connRepo :: Repo
+	, connCheckAuth :: (AuthToken -> Bool)
+	, connIhdl :: Handle
+	, connOhdl :: Handle
 	}
 
+-- Opens a connection to a peer. Does not authenticate with it.
+connectPeer :: Git.Repo -> P2PAddress -> IO P2PConnection
+connectPeer g (TorAnnex onionaddress onionport) = do
+	h <- setupHandle =<< connectHiddenService onionaddress onionport
+	return $ P2PConnection
+		{ connRepo = g
+		, connCheckAuth = const False
+		, connIhdl = h
+		, connOhdl = h
+		}
+
+setupHandle :: Socket -> IO Handle
+setupHandle s = do
+	h <- socketToHandle s ReadWriteMode
+	hSetBuffering h LineBuffering
+	hSetBinaryMode h False
+	fileEncoding h
+	return h
+
 -- Purposefully incomplete interpreter of Proto.
 --
 -- This only runs Net actions. No Local actions will be run
 -- (those need the Annex monad) -- if the interpreter reaches any,
 -- it returns Nothing.
-runNetProto :: RunEnv -> Proto a -> IO (Maybe a)
-runNetProto runenv = go
+runNetProto :: P2PConnection -> Proto a -> IO (Maybe a)
+runNetProto conn = go
   where
 	go :: RunProto IO
 	go (Pure v) = pure (Just v)
-	go (Free (Net n)) = runNet runenv go n
+	go (Free (Net n)) = runNet conn go n
 	go (Free (Local _)) = return Nothing
 
 -- Interpreter of the Net part of Proto.
 --
 -- An interpreter of Proto has to be provided, to handle the rest of Proto
 -- actions.
-runNet :: (MonadIO m, MonadMask m) => RunEnv -> RunProto m -> NetF (Proto a) -> m (Maybe a)
-runNet runenv runner f = case f of
+runNet :: (MonadIO m, MonadMask m) => P2PConnection -> RunProto m -> NetF (Proto a) -> m (Maybe a)
+runNet conn runner f = case f of
 	SendMessage m next -> do
 		v <- liftIO $ tryNonAsync $ do
-			hPutStrLn (runOhdl runenv) (unwords (formatMessage m))
-			hFlush (runOhdl runenv)
+			hPutStrLn (connOhdl conn) (unwords (formatMessage m))
+			hFlush (connOhdl conn)
 		case v of
 			Left _e -> return Nothing
 			Right () -> runner next
 	ReceiveMessage next -> do
-		v <- liftIO $ tryNonAsync $ hGetLine (runIhdl runenv)
+		v <- liftIO $ tryNonAsync $ hGetLine (connIhdl conn)
 		case v of
 			Left _e -> return Nothing
 			Right l -> case parseMessage l of
@@ -81,19 +106,19 @@ runNet runenv runner f = case f of
 					next e
 	SendBytes len b next -> do
 		v <- liftIO $ tryNonAsync $ do
-			ok <- sendExactly len b (runOhdl runenv)
-			hFlush (runOhdl runenv)
+			ok <- sendExactly len b (connOhdl conn)
+			hFlush (connOhdl conn)
 			return ok
 		case v of
 			Right True -> runner next
 			_ -> return Nothing
 	ReceiveBytes (Len n) next -> do
-		v <- liftIO $ tryNonAsync $ L.hGet (runIhdl runenv) (fromIntegral n)
+		v <- liftIO $ tryNonAsync $ L.hGet (connIhdl conn) (fromIntegral n)
 		case v of
 			Left _e -> return Nothing
 			Right b -> runner (next b)
 	CheckAuthToken _u t next -> do
-		let authed = runCheckAuth runenv t
+		let authed = connCheckAuth conn t
 		runner (next authed)
 	Relay hin hout next -> do
 		v <- liftIO $ runRelay runnerio hin hout
@@ -101,7 +126,7 @@ runNet runenv runner f = case f of
 			Nothing -> return Nothing
 			Just exitcode -> runner (next exitcode)
 	RelayService service next -> do
-		v <- liftIO $ runRelayService runenv runnerio service
+		v <- liftIO $ runRelayService conn runnerio service
 		case v of
 			Nothing -> return Nothing
 			Just () -> runner next
@@ -109,7 +134,7 @@ runNet runenv runner f = case f of
 	-- This is only used for running Net actions when relaying,
 	-- so it's ok to use runNetProto, despite it not supporting
 	-- all Proto actions.
-	runnerio = runNetProto runenv
+	runnerio = runNetProto conn
 
 -- Send exactly the specified number of bytes or returns False.
 --
@@ -150,8 +175,8 @@ runRelay runner (RelayHandle hout) (RelayHandle hin) = bracket setup cleanup go
 	
 	go v = relayHelper runner v hin
 
-runRelayService :: RunEnv -> RunProto IO -> Service -> IO (Maybe ())
-runRelayService runenv runner service = bracket setup cleanup go
+runRelayService :: P2PConnection -> RunProto IO -> Service -> IO (Maybe ())
+runRelayService conn runner service = bracket setup cleanup go
   where
 	cmd = case service of
 		UploadPack -> "upload-pack"
@@ -159,8 +184,8 @@ runRelayService runenv runner service = bracket setup cleanup go
 	
 	serviceproc = gitCreateProcess
 		[ Param cmd
-		, File (repoPath (runRepo runenv))
-		] (runRepo runenv)
+		, File (repoPath (connRepo conn))
+		] (connRepo conn)
 
 	setup = do
 		(Just hin, Just hout, _, pid) <- createProcess serviceproc
diff --git a/Remote/Helper/Tor.hs b/Remote/Helper/Tor.hs
deleted file mode 100644
index b5a819c3b7..0000000000
--- a/Remote/Helper/Tor.hs
+++ /dev/null
@@ -1,20 +0,0 @@
-{- Helpers for tor remotes.
- -
- - Copyright 2016 Joey Hess <id@joeyh.name>
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module Remote.Helper.Tor where
-
-import Annex.Common
-
-import Network.Socket
-
-torHandle :: Socket -> IO Handle
-torHandle s = do
-	h <- socketToHandle s ReadWriteMode
-	hSetBuffering h LineBuffering
-	hSetBinaryMode h False
-	fileEncoding h
-	return h
diff --git a/Remote/P2P.hs b/Remote/P2P.hs
index f97d76e71c..0c7ca05748 100644
--- a/Remote/P2P.hs
+++ b/Remote/P2P.hs
@@ -15,14 +15,13 @@ import qualified Annex
 import qualified P2P.Protocol as P2P
 import P2P.Address
 import P2P.Annex
+import P2P.IO
 import Types.Remote
 import Types.GitConfig
 import qualified Git
 import Config
 import Config.Cost
 import Remote.Helper.Git
-import Remote.Helper.Tor
-import Utility.Tor
 import Utility.Metered
 import Types.NumCopies
 
@@ -108,7 +107,7 @@ lock theiruuid addr connpool k callback =
 
 -- | A connection to the peer.
 data Connection 
-	= TorAnnexConnection RunEnv
+	= OpenConnection P2PConnection
 	| ClosedConnection
 
 type ConnectionPool = TVar [Connection]
@@ -122,14 +121,15 @@ runProto addr connpool a = withConnection addr connpool (runProto' a)
 
 runProto' :: P2P.Proto a -> Connection -> Annex (Connection, Maybe a)
 runProto' _ ClosedConnection = return (ClosedConnection, Nothing)
-runProto' a conn@(TorAnnexConnection runenv) = do
-	r <- runFullProto Client runenv a
+runProto' a (OpenConnection conn) = do
+	r <- runFullProto Client conn a
 	-- When runFullProto fails, the connection is no longer usable,
 	-- so close it.
 	if isJust r
-		then return (conn, r)
+		then return (OpenConnection conn, r)
 		else do
-			liftIO $ hClose (runIhdl runenv)
+			liftIO $ hClose (connIhdl conn)
+			liftIO $ hClose (connOhdl conn)
 			return (ClosedConnection, r)
 
 -- Uses an open connection if one is available in the ConnectionPool;
@@ -161,17 +161,9 @@ withConnection addr connpool a = bracketOnError get cache go
 		return r
 
 openConnection :: P2PAddress -> Annex Connection
-openConnection (TorAnnex onionaddress onionport) = do
-	v <- liftIO $ tryNonAsync $ 
-		torHandle =<< connectHiddenService onionaddress onionport
+openConnection addr = do
+	g <- Annex.gitRepo
+	v <- liftIO $ tryNonAsync $ connectPeer g addr
 	case v of
-		Right h -> do
-			g <- Annex.gitRepo
-			let runenv = RunEnv
-				{ runRepo = g
-				, runCheckAuth = const False
-				, runIhdl = h
-				, runOhdl = h
-				}
-			return (TorAnnexConnection runenv)
+		Right conn -> return (OpenConnection conn)
 		Left _e -> return ClosedConnection
diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs
index 2caa7cdb1b..e5d4e97add 100644
--- a/RemoteDaemon/Transport/Tor.hs
+++ b/RemoteDaemon/Transport/Tor.hs
@@ -15,7 +15,6 @@ import RemoteDaemon.Common
 import Utility.Tor
 import Utility.FileMode
 import Utility.AuthToken
-import Remote.Helper.Tor
 import P2P.Protocol
 import P2P.IO
 import P2P.Annex
@@ -55,7 +54,7 @@ server th@(TransportHandle (LocalRepo r) _) = do
 	debugM "remotedaemon" "tor hidden service running"
 	forever $ do
 		(conn, _) <- accept soc
-		h <- torHandle conn
+		h <- setupHandle conn
 		ok <- atomically $ ifM (isFullTBQueue q)
 			( return False
 			, do
@@ -85,16 +84,16 @@ serveClient th u r q = bracket setup cleanup go
 			-- Load auth tokens for every connection, to notice
 			-- when the allowed set is changed.
 			allowed <- loadP2PAuthTokens
-			let runenv = RunEnv
-				{ runRepo = r
-				, runCheckAuth = (`isAllowedAuthToken` allowed)
-				, runIhdl = h
-				, runOhdl = h
+			let conn = P2PConnection
+				{ connRepo = r
+				, connCheckAuth = (`isAllowedAuthToken` allowed)
+				, connIhdl = h
+				, connOhdl = h
 				}
-			v <- liftIO $ runNetProto runenv $ serveAuth u
+			v <- liftIO $ runNetProto conn $ serveAuth u
 			case v of
 				Just (Just theiruuid) -> void $ 
-					runFullProto (Serving theiruuid) runenv $
+					runFullProto (Serving theiruuid) conn $
 						serveAuthed u
 				_ -> return ()
 		-- Merge the duplicated state back in.
diff --git a/git-annex.cabal b/git-annex.cabal
index 7fcba0623c..c894e66104 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -934,7 +934,6 @@ Executable git-annex
     Remote.Helper.ReadOnly
     Remote.Helper.Special
     Remote.Helper.Ssh
-    Remote.Helper.Tor
     Remote.Hook
     Remote.List
     Remote.P2P

From bb5168e8942b288509e7754b1d46c697de8a45e1 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 6 Dec 2016 15:49:39 -0400
Subject: [PATCH 186/367] need to auth with the peer

---
 P2P/IO.hs     |  6 ++++++
 Remote/P2P.hs | 19 ++++++++++++++++---
 2 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/P2P/IO.hs b/P2P/IO.hs
index f63b2808b3..9abefb8a06 100644
--- a/P2P/IO.hs
+++ b/P2P/IO.hs
@@ -11,6 +11,7 @@ module P2P.IO
 	( RunProto
 	, P2PConnection(..)
 	, connectPeer
+	, closeConnection
 	, setupHandle
 	, runNetProto
 	, runNet
@@ -60,6 +61,11 @@ connectPeer g (TorAnnex onionaddress onionport) = do
 		, connOhdl = h
 		}
 
+closeConnection :: P2PConnection -> IO ()
+closeConnection conn = do
+	hClose (connIhdl conn)
+	hClose (connOhdl conn)
+
 setupHandle :: Socket -> IO Handle
 setupHandle s = do
 	h <- socketToHandle s ReadWriteMode
diff --git a/Remote/P2P.hs b/Remote/P2P.hs
index 0c7ca05748..68b75924f3 100644
--- a/Remote/P2P.hs
+++ b/Remote/P2P.hs
@@ -16,13 +16,16 @@ import qualified P2P.Protocol as P2P
 import P2P.Address
 import P2P.Annex
 import P2P.IO
+import P2P.Auth
 import Types.Remote
 import Types.GitConfig
 import qualified Git
+import Annex.UUID
 import Config
 import Config.Cost
 import Remote.Helper.Git
 import Utility.Metered
+import Utility.AuthToken
 import Types.NumCopies
 
 import Control.Concurrent
@@ -128,8 +131,7 @@ runProto' a (OpenConnection conn) = do
 	if isJust r
 		then return (OpenConnection conn, r)
 		else do
-			liftIO $ hClose (connIhdl conn)
-			liftIO $ hClose (connOhdl conn)
+			liftIO $ closeConnection conn
 			return (ClosedConnection, r)
 
 -- Uses an open connection if one is available in the ConnectionPool;
@@ -165,5 +167,16 @@ openConnection addr = do
 	g <- Annex.gitRepo
 	v <- liftIO $ tryNonAsync $ connectPeer g addr
 	case v of
-		Right conn -> return (OpenConnection conn)
+		Right conn -> do
+			myuuid <- getUUID
+			authtoken <- fromMaybe nullAuthToken
+				<$> loadP2PRemoteAuthToken addr
+			res <- liftIO $ runNetProto conn $
+				P2P.auth myuuid authtoken
+			case res of
+				Just (Just _theiruuid) ->
+					return (OpenConnection conn)
+				_ -> do
+					liftIO $ closeConnection conn
+					return ClosedConnection
 		Left _e -> return ClosedConnection

From 60f4b1cf36f0b2e8233e1b48466e1b4106dd1175 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 6 Dec 2016 16:55:53 -0400
Subject: [PATCH 187/367] PAKE

---
 doc/design/assistant/telehash.mdwn | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/design/assistant/telehash.mdwn b/doc/design/assistant/telehash.mdwn
index 5c410999fe..14b617c33b 100644
--- a/doc/design/assistant/telehash.mdwn
+++ b/doc/design/assistant/telehash.mdwn
@@ -123,6 +123,10 @@ so won't want to type that in. Need discovery.
   for Bob to confirm he's ready to finish pairing, this will fail,
   because Bob won't get to that point if the authtoken is intercepted.
 
+  Check out
+  <https://en.wikipedia.org/wiki/Password-authenticated_key_agreement>
+  for more MITM resistance.
+
 ## local lan detection
 
 At connection time, after authentication, the remote can send

From f2354ddef55da2e95e3b3f4aecd263b72cdcf85a Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 6 Dec 2016 17:08:39 -0400
Subject: [PATCH 188/367] devbog

---
 doc/devblog/day_432-433__almost_there.mdwn | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/devblog/day_432-433__almost_there.mdwn

diff --git a/doc/devblog/day_432-433__almost_there.mdwn b/doc/devblog/day_432-433__almost_there.mdwn
new file mode 100644
index 0000000000..b41ce3f701
--- /dev/null
+++ b/doc/devblog/day_432-433__almost_there.mdwn
@@ -0,0 +1,13 @@
+Friday and today were spent implementing both sides of the P2P protocol for
+git-annex content transfers.
+
+There were some tricky cases to deal with. For example, when a file is being
+sent from a direct mode repository, or v6 annex.thin repository, the
+content of the file can change as it's being transferred. Including being
+appended to or truncated. Had to find a way to deal with that, to avoid
+breaking the protocol by not sending the indicated number of bytes of data.
+
+It all seems to be done now, but it's not been tested at all, and there are
+probably some bugs to find. (And progress info is not wired up yet.)
+
+Today's work was sponsored by Trenton Cronholm on Patreon.

From 528c7bee24489e7ca559958808758145b75ab2c2 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 7 Dec 2016 12:00:27 -0400
Subject: [PATCH 189/367] fix up some commands

---
 doc/tips/peer_to_peer_network_with_tor.mdwn | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/tips/peer_to_peer_network_with_tor.mdwn b/doc/tips/peer_to_peer_network_with_tor.mdwn
index 0481874587..43dc0cfc27 100644
--- a/doc/tips/peer_to_peer_network_with_tor.mdwn
+++ b/doc/tips/peer_to_peer_network_with_tor.mdwn
@@ -12,9 +12,9 @@ First, you need to get Tor installed and running. See
 
 To make git-annex use Tor, run these commands in your git-annex repository:
 
-	sudo git annex enable-tor
+	sudo git annex enable-tor $(id -u)
 	git annex remotedaemon
-	git annex p2p --gen-address
+	git annex p2p --gen-addresses
 
 The p2p command will output a long address, such as:
 
@@ -39,7 +39,7 @@ with a new empty repository:
 And make git-annex use Tor, by running these commands in the git-annex
 repository:
 
-	sudo git annex enable-tor
+	sudo git annex enable-tor $(id -u)
 	git annex remotedaemon
 
 Now, tell the new peer about the address of the first peer.
@@ -57,7 +57,7 @@ peer1 remote:
 	git annex sync --content peer1
 
 You can also generate an address for this new peer, by running `git annex
-p2p --gen-address`, and link other peers to that address using `git annex
+p2p --gen-addresses`, and link other peers to that address using `git annex
 p2p --link`. It's often useful to link peers up in both directions,
 so peer1 is a remote of peer2 and peer2 is a remote of peer1.
 
@@ -94,7 +94,7 @@ that git-annex uses to make sure that only people you want to can access
 your repository over Tor. That takes the form of a long string of numbers
 and letters, like "7f53c5b65b8957ef626fd461ceaae8056e3dbc459ae715e4".
 
-The addresses generated by `git annex peer --gen-address` 
+The addresses generated by `git annex peer --gen-addresses` 
 combine the onion address with the authentication data.
 
 When you run `git annex peer --link`, it sets up a git remote using

From 729e864ad0ea8f6cd39c12f8d1e2424bf5d0e4fa Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 7 Dec 2016 12:11:38 -0400
Subject: [PATCH 190/367] add todo list for remaining tor stuff

---
 doc/todo/tor.mdwn | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 doc/todo/tor.mdwn

diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
new file mode 100644
index 0000000000..d97c0a986c
--- /dev/null
+++ b/doc/todo/tor.mdwn
@@ -0,0 +1,20 @@
+git-annex sync over tor
+
+Mostly working!
+
+Current todo list:
+
+* uuid discovery in p2p link
+* copy --to peer seems to make the remotedaemon buffer the content in
+  memory, more than I'd expect.
+* update progress meters
+* Think about locking some more. What happens if the connection to the peer
+  is dropped while we think we're locking content there from being dropped?
+* merge to master
+
+Eventually:
+
+* address exchange via electrum-mnemonic or magic wormhole (see PAKE)
+* webapp UI for easy pairing
+* friend-of-a-friend peer discovery to build more interconnected networks
+  of nodes

From 0d9a11625cd2b0463a88442ab67170b7d817ed4d Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 7 Dec 2016 12:38:21 -0400
Subject: [PATCH 191/367] remote uuid discovery in p2p --link

This also tests that we can connect to the peer.

This commit was sponsored by Jeff Goeke-Smith on Patreon.
---
 Command/P2P.hs    | 30 +++++++++++++++++++++++-------
 doc/todo/tor.mdwn |  1 -
 2 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/Command/P2P.hs b/Command/P2P.hs
index e2a7ab85d2..21632f3da5 100644
--- a/Command/P2P.hs
+++ b/Command/P2P.hs
@@ -10,10 +10,15 @@ module Command.P2P where
 import Command
 import P2P.Address
 import P2P.Auth
+import P2P.IO
+import qualified P2P.Protocol as P2P
 import Utility.AuthToken
 import Git.Types
 import qualified Git.Remote
 import qualified Git.Command
+import qualified Annex
+import Annex.UUID
+import Config
 
 cmd :: Command
 cmd = command "p2p" SectionSetup
@@ -61,7 +66,7 @@ linkRemote remotename = do
   where
 	prompt = do
 		liftIO $ putStrLn ""
-		liftIO $ putStr "Enter address: "
+		liftIO $ putStr "Enter peer address: "
 		liftIO $ hFlush stdout
 		s <- liftIO getLine
 		if null s
@@ -74,9 +79,20 @@ linkRemote remotename = do
 					prompt
 				Just addr -> setup addr
 	setup (P2PAddressAuth addr authtoken) = do
-		storeP2PRemoteAuthToken addr authtoken
-		inRepo $ Git.Command.runBool
-			[ Param "remote", Param "add"
-			, Param remotename
-			, Param (formatP2PAddress addr)
-			]
+		g <- Annex.gitRepo
+		conn <- liftIO $ connectPeer g addr
+			`catchNonAsync` giveup "Unable to connect with peer. Please check that the peer is connected to the network, and try again."
+		u <- getUUID
+		v <- liftIO $ runNetProto conn $ P2P.auth u authtoken
+		case v of
+			Just (Just theiruuid) -> do
+				ok <- inRepo $ Git.Command.runBool
+					[ Param "remote", Param "add"
+					, Param remotename
+					, Param (formatP2PAddress addr)
+					]
+				when ok $ do
+					storeUUIDIn (remoteConfig remotename "uuid") theiruuid
+					storeP2PRemoteAuthToken addr authtoken
+				return ok
+			_ -> giveup "Unable to authenticate with peer. Please check the address and try again."
diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index d97c0a986c..b1a4f8f542 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -4,7 +4,6 @@ Mostly working!
 
 Current todo list:
 
-* uuid discovery in p2p link
 * copy --to peer seems to make the remotedaemon buffer the content in
   memory, more than I'd expect.
 * update progress meters

From 757d36f8ca7efdcca236901d756c9e1fbb5139aa Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 7 Dec 2016 12:39:28 -0400
Subject: [PATCH 192/367] validate peer uuid each time we talk to it

In case the repo on the peer changes uuid (eg by a new repo being moved
into place).

Also, added some warning messages when unable to communicate with a
peer.

This commit was sponsored by Anthony DeRobertis on Patreon.
---
 Remote/P2P.hs | 65 ++++++++++++++++++++++++++++-----------------------
 1 file changed, 36 insertions(+), 29 deletions(-)

diff --git a/Remote/P2P.hs b/Remote/P2P.hs
index 68b75924f3..f4f1d5f38f 100644
--- a/Remote/P2P.hs
+++ b/Remote/P2P.hs
@@ -49,12 +49,12 @@ chainGen addr r u c gc = do
 		{ uuid = u
 		, cost = cst
 		, name = Git.repoDescribe r
-		, storeKey = store addr connpool
-		, retrieveKeyFile = retrieve addr connpool
+		, storeKey = store u addr connpool
+		, retrieveKeyFile = retrieve u addr connpool
 		, retrieveKeyFileCheap = \_ _ _ -> return False
-		, removeKey = remove addr connpool
+		, removeKey = remove u addr connpool
 		, lockContent = Just (lock u addr connpool)
-		, checkPresent = checkpresent addr connpool
+		, checkPresent = checkpresent u addr connpool
 		, checkPresentCheap = False
 		, whereisKey = Nothing
 		, remoteFsck = Nothing
@@ -74,27 +74,27 @@ chainGen addr r u c gc = do
 	return (Just this)
 
 -- TODO update progress
-store :: P2PAddress -> ConnectionPool -> Key -> AssociatedFile -> MeterUpdate -> Annex Bool
-store addr connpool k af p = fromMaybe False
-	<$> runProto addr connpool (P2P.put k af)
+store :: UUID -> P2PAddress -> ConnectionPool -> Key -> AssociatedFile -> MeterUpdate -> Annex Bool
+store u addr connpool k af p = fromMaybe False
+	<$> runProto u addr connpool (P2P.put k af)
 
-retrieve :: P2PAddress -> ConnectionPool -> Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex (Bool, Verification)
-retrieve addr connpool k af dest _p = unVerified $ fromMaybe False 
-	<$> runProto addr connpool (P2P.get dest k af)
+retrieve :: UUID -> P2PAddress -> ConnectionPool -> Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex (Bool, Verification)
+retrieve u addr connpool k af dest _p = unVerified $ fromMaybe False 
+	<$> runProto u addr connpool (P2P.get dest k af)
 
-remove :: P2PAddress -> ConnectionPool -> Key -> Annex Bool
-remove addr connpool k = fromMaybe False
-	<$> runProto addr connpool (P2P.remove k)
+remove :: UUID -> P2PAddress -> ConnectionPool -> Key -> Annex Bool
+remove u addr connpool k = fromMaybe False
+	<$> runProto u addr connpool (P2P.remove k)
 
-checkpresent :: P2PAddress -> ConnectionPool -> Key -> Annex Bool
-checkpresent addr connpool k = maybe unavail return
-	=<< runProto addr connpool (P2P.checkPresent k)
+checkpresent :: UUID -> P2PAddress -> ConnectionPool -> Key -> Annex Bool
+checkpresent u addr connpool k = maybe unavail return
+	=<< runProto u addr connpool (P2P.checkPresent k)
   where
 	unavail = giveup "can't connect to peer"
 
 lock :: UUID -> P2PAddress -> ConnectionPool -> Key -> (VerifiedCopy -> Annex r) -> Annex r
-lock theiruuid addr connpool k callback =
-	withConnection addr connpool $ \conn -> do
+lock u addr connpool k callback =
+	withConnection u addr connpool $ \conn -> do
 		connv <- liftIO $ newMVar conn
 		let runproto d p = do
 			c <- liftIO $ takeMVar connv
@@ -106,7 +106,7 @@ lock theiruuid addr connpool k callback =
 		return (conn', r)
   where
 	go False = giveup "can't lock content"
-	go True = withVerifiedCopy LockedCopy theiruuid (return True) callback
+	go True = withVerifiedCopy LockedCopy u (return True) callback
 
 -- | A connection to the peer.
 data Connection 
@@ -119,8 +119,8 @@ mkConnectionPool :: Annex ConnectionPool
 mkConnectionPool = liftIO $ newTVarIO []
 
 -- Runs the Proto action.
-runProto :: P2PAddress -> ConnectionPool -> P2P.Proto a -> Annex (Maybe a)
-runProto addr connpool a = withConnection addr connpool (runProto' a)
+runProto :: UUID -> P2PAddress -> ConnectionPool -> P2P.Proto a -> Annex (Maybe a)
+runProto u addr connpool a = withConnection u addr connpool (runProto' a)
 
 runProto' :: P2P.Proto a -> Connection -> Annex (Connection, Maybe a)
 runProto' _ ClosedConnection = return (ClosedConnection, Nothing)
@@ -139,8 +139,8 @@ runProto' a (OpenConnection conn) = do
 --
 -- Once the action is done, the connection is added back to the
 -- ConnectionPool, unless it's no longer open.
-withConnection :: P2PAddress -> ConnectionPool -> (Connection -> Annex (Connection, a)) -> Annex a
-withConnection addr connpool a = bracketOnError get cache go
+withConnection :: UUID -> P2PAddress -> ConnectionPool -> (Connection -> Annex (Connection, a)) -> Annex a
+withConnection u addr connpool a = bracketOnError get cache go
   where
 	get = do
 		mc <- liftIO $ atomically $ do
@@ -152,7 +152,7 @@ withConnection addr connpool a = bracketOnError get cache go
 				(c:cs) -> do
 					writeTVar connpool cs
 					return (Just c)
-		maybe (openConnection addr) return mc
+		maybe (openConnection u addr) return mc
 	
 	cache ClosedConnection = return ()
 	cache conn = liftIO $ atomically $ modifyTVar' connpool (conn:)
@@ -162,8 +162,8 @@ withConnection addr connpool a = bracketOnError get cache go
 		cache conn'
 		return r
 
-openConnection :: P2PAddress -> Annex Connection
-openConnection addr = do
+openConnection :: UUID -> P2PAddress -> Annex Connection
+openConnection u addr = do
 	g <- Annex.gitRepo
 	v <- liftIO $ tryNonAsync $ connectPeer g addr
 	case v of
@@ -174,9 +174,16 @@ openConnection addr = do
 			res <- liftIO $ runNetProto conn $
 				P2P.auth myuuid authtoken
 			case res of
-				Just (Just _theiruuid) ->
-					return (OpenConnection conn)
+				Just (Just theiruuid)
+					| u == theiruuid -> return (OpenConnection conn)
+					| otherwise -> do
+						liftIO $ closeConnection conn
+						warning "Remote peer uuid seems to have changed."
+						return ClosedConnection
 				_ -> do
 					liftIO $ closeConnection conn
+					warning "Unable to authenticate with peer."
 					return ClosedConnection
-		Left _e -> return ClosedConnection
+		Left _e -> do
+			warning "Unable to connect to peer."
+			return ClosedConnection

From 7c245b2180e53346caa97eb2405e17e8818a91ce Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 7 Dec 2016 12:48:24 -0400
Subject: [PATCH 193/367] update

---
 doc/design/assistant/telehash.mdwn | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/doc/design/assistant/telehash.mdwn b/doc/design/assistant/telehash.mdwn
index 14b617c33b..6a098ba515 100644
--- a/doc/design/assistant/telehash.mdwn
+++ b/doc/design/assistant/telehash.mdwn
@@ -66,18 +66,17 @@ or [cjdns](https://github.com/cjdelisle/cjdns) or tor or i2p or [magic wormhole]
 
 ## general design
 
-* Make address.log that contains (uuid, transport, address, Maybe authtoken)
-* The authtoken is an additional guard, to protect against transports
-  where the address might be able to be guessed, or observed by the rest of
-  the network.
-* Some addresses can be used with only the provided authtoken
-  from the address.log. Remotes can be auto-enabled for these.
-* Other addresses have Nothing povided for the authtoken, and one
-  has to instead be provided during manual enabling of the remote.
+* There is a generic P2P protocol, which should be usable with any P2P
+  system that can send messages between peers.
+* A p2p remote has an url like tor-annex::fijdksajdksjfkj, which connects
+  to a specific peer. The peer's address may be kept private, but 
+  the design allows the address to be public without giving access to
+  the peer.
+* An authtoken also needs to be presented when connecting with a peer.
+  This is stored in local creds storage and must be kept private.
 * The remotedaemon runs, and/or communicates with the program implementing
-  the network transport. For example for tor, the remotedaemon runs
-  the hidden service, and also connects to the tor hidden services of
-  other nodes.
+  the P2P network. For example for tor, the remotedaemon runs the
+  hidden service.
 * The remotedaemon handles both sides of git push over the transport.
 * The remotedaemon may also support sending objects over the transport,
   depending on the transport.

From 83ea1cec86f7e09f04cd882fe52899f87ccb193b Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 7 Dec 2016 13:37:35 -0400
Subject: [PATCH 194/367] update progress meter when sending to p2p remote

This commit was sponsored by Thom May on Patreon.
---
 P2P/IO.hs          | 23 ++++++++---------------
 P2P/Protocol.hs    | 19 ++++++++++---------
 Remote/P2P.hs      |  3 +--
 Utility/Metered.hs |  7 +++++--
 4 files changed, 24 insertions(+), 28 deletions(-)

diff --git a/P2P/IO.hs b/P2P/IO.hs
index 9abefb8a06..8a580452c0 100644
--- a/P2P/IO.hs
+++ b/P2P/IO.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-{-# LANGUAGE RankNTypes, FlexibleContexts, BangPatterns, CPP #-}
+{-# LANGUAGE RankNTypes, FlexibleContexts, CPP #-}
 
 module P2P.IO
 	( RunProto
@@ -26,6 +26,7 @@ import Utility.AuthToken
 import Utility.SafeCommand
 import Utility.SimpleProtocol
 import Utility.Exception
+import Utility.Metered
 import Utility.Tor
 import Utility.FileSystemEncoding
 
@@ -110,9 +111,9 @@ runNet conn runner f = case f of
 					let e = ERROR $ "protocol parse error: " ++ show l
 					net $ sendMessage e
 					next e
-	SendBytes len b next -> do
+	SendBytes len b p next -> do
 		v <- liftIO $ tryNonAsync $ do
-			ok <- sendExactly len b (connOhdl conn)
+			ok <- sendExactly len b (connOhdl conn) p
 			hFlush (connOhdl conn)
 			return ok
 		case v of
@@ -153,18 +154,10 @@ runNet conn runner f = case f of
 --
 -- If too few bytes are sent, the only option is to give up on this
 -- connection. False is returned to indicate this problem.
---
--- We can't check the length of the whole lazy bytestring without buffering
--- it in memory. Instead, process it one chunk at a time, and sum the length
--- of the chunks.
-sendExactly :: Len -> L.ByteString -> Handle -> IO Bool
-sendExactly (Len l) lb h = go 0 $ L.toChunks $ L.take (fromIntegral l) lb
-  where
-	go n [] = return (toInteger n == l)
-	go n (b:bs) = do
-		B.hPut h b
-		let !n' = n + B.length b
-		go n' bs
+sendExactly :: Len -> L.ByteString -> Handle -> MeterUpdate -> IO Bool
+sendExactly (Len l) b h p = do
+	sent <- meteredWrite' p h (L.take (fromIntegral l) b)
+	return (fromBytesProcessed sent == l)
 
 runRelay :: RunProto IO -> RelayHandle -> RelayHandle -> IO (Maybe ExitCode)
 runRelay runner (RelayHandle hout) (RelayHandle hin) = bracket setup cleanup go
diff --git a/P2P/Protocol.hs b/P2P/Protocol.hs
index b2b734e48a..6cefce38c0 100644
--- a/P2P/Protocol.hs
+++ b/P2P/Protocol.hs
@@ -17,6 +17,7 @@ import Types.UUID
 import Utility.AuthToken
 import Utility.Applicative
 import Utility.PartialPrelude
+import Utility.Metered
 import Git.FilePath
 
 import Control.Monad
@@ -163,7 +164,7 @@ local = hoistFree Local
 data NetF c
 	= SendMessage Message c
 	| ReceiveMessage (Message -> c)
-	| SendBytes Len L.ByteString c
+	| SendBytes Len L.ByteString MeterUpdate c
 	-- ^ Sends exactly Len bytes of data. (Any more or less will
 	-- confuse the receiver.)
 	| ReceiveBytes Len (L.ByteString -> c)
@@ -278,12 +279,12 @@ get dest key af = receiveContent sizer storer (\offset -> GET offset af key)
 	sizer = fileSize dest
 	storer = storeContentTo dest
 
-put :: Key -> AssociatedFile -> Proto Bool
-put key af = do
+put :: Key -> AssociatedFile -> MeterUpdate -> Proto Bool
+put key af p = do
 	net $ sendMessage (PUT af key)
 	r <- net receiveMessage
 	case r of
-		PUT_FROM offset -> sendContent key af offset
+		PUT_FROM offset -> sendContent key af offset p
 		ALREADY_HAVE -> return True
 		_ -> do
 			net $ sendMessage (ERROR "expected PUT_FROM")
@@ -368,7 +369,7 @@ serveAuthed myuuid = void $ serverLoop handler
 					local $ setPresent key myuuid
 		return ServerContinue
 	handler (GET offset key af) = do
-		void $ sendContent af key offset
+		void $ sendContent af key offset nullMeterUpdate
 		-- setPresent not called because the peer may have
 		-- requested the data but not permanently stored it.
 		return ServerContinue
@@ -377,11 +378,11 @@ serveAuthed myuuid = void $ serverLoop handler
 		return ServerContinue
 	handler _ = return ServerUnexpected
 
-sendContent :: Key -> AssociatedFile -> Offset -> Proto Bool
-sendContent key af offset = do
+sendContent :: Key -> AssociatedFile -> Offset -> MeterUpdate -> Proto Bool
+sendContent key af offset p = do
 	(len, content) <- readContentLen key af offset
 	net $ sendMessage (DATA len)
-	net $ sendBytes len content
+	net $ sendBytes len content p
 	checkSuccess
 
 receiveContent :: Local Len -> (Offset -> Len -> L.ByteString -> Local Bool) -> (Offset -> Message) -> Proto Bool
@@ -456,5 +457,5 @@ relayToPeer (RelayDone exitcode) = sendMessage (CONNECTDONE exitcode)
 relayToPeer (RelayToPeer b) = do
 	let len = Len $ fromIntegral $ L.length b
 	sendMessage (DATA len)
-	sendBytes len b
+	sendBytes len b nullMeterUpdate
 relayToPeer (RelayFromPeer _) = return ()
diff --git a/Remote/P2P.hs b/Remote/P2P.hs
index f4f1d5f38f..8286a9a18c 100644
--- a/Remote/P2P.hs
+++ b/Remote/P2P.hs
@@ -73,10 +73,9 @@ chainGen addr r u c gc = do
 	}
 	return (Just this)
 
--- TODO update progress
 store :: UUID -> P2PAddress -> ConnectionPool -> Key -> AssociatedFile -> MeterUpdate -> Annex Bool
 store u addr connpool k af p = fromMaybe False
-	<$> runProto u addr connpool (P2P.put k af)
+	<$> runProto u addr connpool (P2P.put k af p)
 
 retrieve :: UUID -> P2PAddress -> ConnectionPool -> Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex (Bool, Verification)
 retrieve u addr connpool k af dest _p = unVerified $ fromMaybe False 
diff --git a/Utility/Metered.hs b/Utility/Metered.hs
index 440aa3f074..aa65efd4d4 100644
--- a/Utility/Metered.hs
+++ b/Utility/Metered.hs
@@ -85,9 +85,12 @@ streamMeteredFile f meterupdate h = withMeteredFile f meterupdate $ L.hPut h
 
 {- Writes a ByteString to a Handle, updating a meter as it's written. -}
 meteredWrite :: MeterUpdate -> Handle -> L.ByteString -> IO ()
-meteredWrite meterupdate h = go zeroBytesProcessed . L.toChunks 
+meteredWrite meterupdate h = void . meteredWrite' meterupdate h
+
+meteredWrite' :: MeterUpdate -> Handle -> L.ByteString -> IO BytesProcessed
+meteredWrite' meterupdate h = go zeroBytesProcessed . L.toChunks 
   where
-	go _ [] = return ()
+	go sofar [] = return sofar
 	go sofar (c:cs) = do
 		S.hPut h c
 		let sofar' = addBytesProcessed sofar $ S.length c

From f3a3dc14ecde51ae817b87a7739cf5f3870ca923 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 7 Dec 2016 13:58:10 -0400
Subject: [PATCH 195/367] add section on security

---
 doc/tips/peer_to_peer_network_with_tor.mdwn | 36 +++++++++++++++++++--
 1 file changed, 33 insertions(+), 3 deletions(-)

diff --git a/doc/tips/peer_to_peer_network_with_tor.mdwn b/doc/tips/peer_to_peer_network_with_tor.mdwn
index 43dc0cfc27..718a9218d3 100644
--- a/doc/tips/peer_to_peer_network_with_tor.mdwn
+++ b/doc/tips/peer_to_peer_network_with_tor.mdwn
@@ -1,9 +1,9 @@
 git-annex has recently gotten support for running as a
-[Tor](http://http://torproject.org/) hidden service. This is a great, and
-very secure way to connect repositories between computers in different
+[Tor](http://http://torproject.org/) hidden service. This is a nice secure
+and easy to use way to connect repositories between peers in different
 locations, without needing any central server.
 
-## the first peer
+## setting up the first peer
 
 First, you need to get Tor installed and running. See
 [their website](http://http://torproject.org/), or try a command like:
@@ -100,3 +100,33 @@ combine the onion address with the authentication data.
 When you run `git annex peer --link`, it sets up a git remote using
 the onion address, and it stashes the authentication data away in a file in
 `.git/annex/creds/`
+
+## security
+
+Tor hidden services can be quite secure. But this doesn't mean that using
+git-annex over Tor is automatically perfectly secure. Here are some things
+to consider:
+
+* Anyone who learns the address of a peer can connect to that peer,
+  download the whole history of the git repository, and any available
+  annexed files. They can also upload new files to the peer, and even
+  remove annexed files from the peer. So consider ways that the address
+  of a peer might be exposed.
+
+* While Tor can be used to anonymize who you are, git defaults to including
+  your name and email address in git commit messages. So if you want an
+  anonymous git-annex repository, you'll need to configure git not to do
+  that.
+
+* Using Tor prevents listeners from decrypting your traffic. But, they'll
+  probably still know you're using Tor. Also, by traffic analysis,
+  they may be able to guess if you're using git-annex over tor, and even
+  make guesses about the sizes and types of files that you're exchanging
+  with peers.
+
+* There have been past attacks on the Tor network that have exposed
+  who was running Tor hidden services.
+  <https://blog.torproject.org/blog/tor-security-advisory-relay-early-traffic-confirmation-attack>
+
+* An attacker who can connect to the git-annex Tor hidden service, even
+  without authenticating, can try to perform denial of service attacks.

From ad5ef51040f4220dd27cb8bfd2951ca1145c8c0e Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 7 Dec 2016 14:25:01 -0400
Subject: [PATCH 196/367] more p2p progress meters

Display progress meter on send and receive from remote.

Added a new hGetMetered that can read an exact number of bytes (or
less), updating a meter as it goes.

This commit was sponsored by Andreas on Patreon.
---
 P2P/IO.hs          | 13 ++++++++-----
 P2P/Protocol.hs    | 16 ++++++++--------
 Remote/P2P.hs      | 11 +++++++----
 Remote/S3.hs       |  2 +-
 Utility/Metered.hs | 30 +++++++++++++++++++++---------
 5 files changed, 45 insertions(+), 27 deletions(-)

diff --git a/P2P/IO.hs b/P2P/IO.hs
index 8a580452c0..ea15ecfc3a 100644
--- a/P2P/IO.hs
+++ b/P2P/IO.hs
@@ -119,8 +119,8 @@ runNet conn runner f = case f of
 		case v of
 			Right True -> runner next
 			_ -> return Nothing
-	ReceiveBytes (Len n) next -> do
-		v <- liftIO $ tryNonAsync $ L.hGet (connIhdl conn) (fromIntegral n)
+	ReceiveBytes len p next -> do
+		v <- liftIO $ tryNonAsync $ receiveExactly len (connIhdl conn) p
 		case v of
 			Left _e -> return Nothing
 			Right b -> runner (next b)
@@ -155,9 +155,12 @@ runNet conn runner f = case f of
 -- If too few bytes are sent, the only option is to give up on this
 -- connection. False is returned to indicate this problem.
 sendExactly :: Len -> L.ByteString -> Handle -> MeterUpdate -> IO Bool
-sendExactly (Len l) b h p = do
-	sent <- meteredWrite' p h (L.take (fromIntegral l) b)
-	return (fromBytesProcessed sent == l)
+sendExactly (Len n) b h p = do
+	sent <- meteredWrite' p h (L.take (fromIntegral n) b)
+	return (fromBytesProcessed sent == n)
+
+receiveExactly :: Len -> Handle -> MeterUpdate -> IO L.ByteString
+receiveExactly (Len n) h p = hGetMetered h (Just n) p
 
 runRelay :: RunProto IO -> RelayHandle -> RelayHandle -> IO (Maybe ExitCode)
 runRelay runner (RelayHandle hout) (RelayHandle hin) = bracket setup cleanup go
diff --git a/P2P/Protocol.hs b/P2P/Protocol.hs
index 6cefce38c0..b1e2bf4817 100644
--- a/P2P/Protocol.hs
+++ b/P2P/Protocol.hs
@@ -167,7 +167,7 @@ data NetF c
 	| SendBytes Len L.ByteString MeterUpdate c
 	-- ^ Sends exactly Len bytes of data. (Any more or less will
 	-- confuse the receiver.)
-	| ReceiveBytes Len (L.ByteString -> c)
+	| ReceiveBytes Len MeterUpdate (L.ByteString -> c)
 	-- ^ Lazily reads bytes from peer. Stops once Len are read,
 	-- or if connection is lost, and in either case returns the bytes
 	-- that were read. This allows resuming interrupted transfers.
@@ -273,8 +273,8 @@ remove key = do
 	net $ sendMessage (REMOVE key)
 	checkSuccess
 
-get :: FilePath -> Key -> AssociatedFile -> Proto Bool
-get dest key af = receiveContent sizer storer (\offset -> GET offset af key)
+get :: FilePath -> Key -> AssociatedFile -> MeterUpdate -> Proto Bool
+get dest key af p = receiveContent p sizer storer (\offset -> GET offset af key)
   where
 	sizer = fileSize dest
 	storer = storeContentTo dest
@@ -364,7 +364,7 @@ serveAuthed myuuid = void $ serverLoop handler
 			else do
 				let sizer = tmpContentSize key
 				let storer = storeContent key af
-				ok <- receiveContent sizer storer PUT_FROM
+				ok <- receiveContent nullMeterUpdate sizer storer PUT_FROM
 				when ok $
 					local $ setPresent key myuuid
 		return ServerContinue
@@ -385,8 +385,8 @@ sendContent key af offset p = do
 	net $ sendBytes len content p
 	checkSuccess
 
-receiveContent :: Local Len -> (Offset -> Len -> L.ByteString -> Local Bool) -> (Offset -> Message) -> Proto Bool
-receiveContent sizer storer mkmsg = do
+receiveContent :: MeterUpdate -> Local Len -> (Offset -> Len -> L.ByteString -> Local Bool) -> (Offset -> Message) -> Proto Bool
+receiveContent p sizer storer mkmsg = do
 	Len n <- local sizer
 	let offset = Offset n
 	net $ sendMessage (mkmsg offset)
@@ -394,7 +394,7 @@ receiveContent sizer storer mkmsg = do
 	case r of
 		DATA len -> do
 			ok <- local . storer offset len
-				=<< net (receiveBytes len)
+				=<< net (receiveBytes len p)
 			sendSuccess ok
 			return ok
 		_ -> do
@@ -447,7 +447,7 @@ relayFromPeer = do
 	r <- receiveMessage
 	case r of
 		CONNECTDONE exitcode -> return $ RelayDone exitcode
-		DATA len -> RelayFromPeer <$> receiveBytes len
+		DATA len -> RelayFromPeer <$> receiveBytes len nullMeterUpdate
 		_ -> do
 			sendMessage $ ERROR "expected DATA or CONNECTDONE"
 			return $ RelayDone $ ExitFailure 1
diff --git a/Remote/P2P.hs b/Remote/P2P.hs
index 8286a9a18c..1d7ede30ff 100644
--- a/Remote/P2P.hs
+++ b/Remote/P2P.hs
@@ -24,6 +24,7 @@ import Annex.UUID
 import Config
 import Config.Cost
 import Remote.Helper.Git
+import Messages.Progress
 import Utility.Metered
 import Utility.AuthToken
 import Types.NumCopies
@@ -74,12 +75,14 @@ chainGen addr r u c gc = do
 	return (Just this)
 
 store :: UUID -> P2PAddress -> ConnectionPool -> Key -> AssociatedFile -> MeterUpdate -> Annex Bool
-store u addr connpool k af p = fromMaybe False
-	<$> runProto u addr connpool (P2P.put k af p)
+store u addr connpool k af p = 
+	metered (Just p) k $ \p' -> fromMaybe False
+		<$> runProto u addr connpool (P2P.put k af p')
 
 retrieve :: UUID -> P2PAddress -> ConnectionPool -> Key -> AssociatedFile -> FilePath -> MeterUpdate -> Annex (Bool, Verification)
-retrieve u addr connpool k af dest _p = unVerified $ fromMaybe False 
-	<$> runProto u addr connpool (P2P.get dest k af)
+retrieve u addr connpool k af dest p = unVerified $ 
+	metered (Just p) k $ \p' -> fromMaybe False 
+		<$> runProto u addr connpool (P2P.get dest k af p')
 
 remove :: UUID -> P2PAddress -> ConnectionPool -> Key -> Annex Bool
 remove u addr connpool k = fromMaybe False
diff --git a/Remote/S3.hs b/Remote/S3.hs
index c6f23333f1..4c1bd5784f 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -193,7 +193,7 @@ store _r info h = fileStorer $ \k f p -> do
 		uploadid <- S3.imurUploadId <$> sendS3Handle h startreq
 
 		-- The actual part size will be a even multiple of the
-		-- 32k chunk size that hGetUntilMetered uses.
+		-- 32k chunk size that lazy ByteStrings use.
 		let partsz' = (partsz `div` toInteger defaultChunkSize) * toInteger defaultChunkSize
 
 		-- Send parts of the file, taking care to stream each part
diff --git a/Utility/Metered.hs b/Utility/Metered.hs
index aa65efd4d4..b80d3ae3f9 100644
--- a/Utility/Metered.hs
+++ b/Utility/Metered.hs
@@ -1,6 +1,6 @@
 {- Metered IO and actions
  -
- - Copyright 2012-2106 Joey Hess <id@joeyh.name>
+ - Copyright 2012-2016 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}
@@ -115,24 +115,24 @@ offsetMeterUpdate base offset = \n -> base (offset `addBytesProcessed` n)
  - meter updates, so use caution.
  -}
 hGetContentsMetered :: Handle -> MeterUpdate -> IO L.ByteString
-hGetContentsMetered h = hGetUntilMetered h (const True)
+hGetContentsMetered h = hGetMetered h Nothing
 
-{- Reads from the Handle, updating the meter after each chunk.
+{- Reads from the Handle, updating the meter after each chunk is read.
+ -
+ - Stops at EOF, or when the requested number of bytes have been read.
+ - Closes the Handle at EOF, but otherwise leaves it open.
  -
  - Note that the meter update is run in unsafeInterleaveIO, which means that
  - it can be run at any time. It's even possible for updates to run out
  - of order, as different parts of the ByteString are consumed.
- -
- - Stops at EOF, or when keepgoing evaluates to False.
- - Closes the Handle at EOF, but otherwise leaves it open.
  -}
-hGetUntilMetered :: Handle -> (Integer -> Bool) -> MeterUpdate -> IO L.ByteString
-hGetUntilMetered h keepgoing meterupdate = lazyRead zeroBytesProcessed
+hGetMetered :: Handle -> Maybe Integer -> MeterUpdate -> IO L.ByteString
+hGetMetered h wantsize meterupdate = lazyRead zeroBytesProcessed
   where
 	lazyRead sofar = unsafeInterleaveIO $ loop sofar
 
 	loop sofar = do
-		c <- S.hGet h defaultChunkSize
+		c <- S.hGet h (nextchunksize (fromBytesProcessed sofar))
 		if S.null c
 			then do
 				hClose h
@@ -148,6 +148,18 @@ hGetUntilMetered h keepgoing meterupdate = lazyRead zeroBytesProcessed
 						cs <- lazyRead sofar'
 						return $ L.append (L.fromChunks [c]) cs
 					else return $ L.fromChunks [c]
+	
+	keepgoing n = case wantsize of
+		Nothing -> True
+		Just sz -> n < sz
+	
+	nextchunksize n = case wantsize of
+		Nothing -> defaultChunkSize
+		Just sz -> 
+			let togo = sz - n
+			in if togo < toInteger defaultChunkSize
+				then fromIntegral togo
+				else defaultChunkSize
 
 {- Same default chunk size Lazy ByteStrings use. -}
 defaultChunkSize :: Int

From 5ff85c8515562e9cfc597d0f27e8167a52416c7d Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 7 Dec 2016 14:46:27 -0400
Subject: [PATCH 197/367] todo

---
 doc/todo/tor.mdwn | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index b1a4f8f542..1432c770e3 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -6,7 +6,9 @@ Current todo list:
 
 * copy --to peer seems to make the remotedaemon buffer the content in
   memory, more than I'd expect.
-* update progress meters
+* update progress logs in remotedaemon send/receive
+* Resuming an interrupted download does not jump the progress to reflect
+  the amount already present. And, it fails at the end.
 * Think about locking some more. What happens if the connection to the peer
   is dropped while we think we're locking content there from being dropped?
 * merge to master

From b55399e3ac23845826b32386b42f36a7a2644368 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 7 Dec 2016 14:52:10 -0400
Subject: [PATCH 198/367] offset meters when resuming

---
 P2P/Protocol.hs   | 8 +++++---
 doc/todo/tor.mdwn | 3 +--
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/P2P/Protocol.hs b/P2P/Protocol.hs
index b1e2bf4817..f51ea7f98c 100644
--- a/P2P/Protocol.hs
+++ b/P2P/Protocol.hs
@@ -379,22 +379,24 @@ serveAuthed myuuid = void $ serverLoop handler
 	handler _ = return ServerUnexpected
 
 sendContent :: Key -> AssociatedFile -> Offset -> MeterUpdate -> Proto Bool
-sendContent key af offset p = do
+sendContent key af offset@(Offset n) p = do
+	let p' = offsetMeterUpdate p (toBytesProcessed n)
 	(len, content) <- readContentLen key af offset
 	net $ sendMessage (DATA len)
-	net $ sendBytes len content p
+	net $ sendBytes len content p'
 	checkSuccess
 
 receiveContent :: MeterUpdate -> Local Len -> (Offset -> Len -> L.ByteString -> Local Bool) -> (Offset -> Message) -> Proto Bool
 receiveContent p sizer storer mkmsg = do
 	Len n <- local sizer
+	let p' = offsetMeterUpdate p (toBytesProcessed n)
 	let offset = Offset n
 	net $ sendMessage (mkmsg offset)
 	r <- net receiveMessage
 	case r of
 		DATA len -> do
 			ok <- local . storer offset len
-				=<< net (receiveBytes len p)
+				=<< net (receiveBytes len p')
 			sendSuccess ok
 			return ok
 		_ -> do
diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index 1432c770e3..af4fa19939 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -7,8 +7,7 @@ Current todo list:
 * copy --to peer seems to make the remotedaemon buffer the content in
   memory, more than I'd expect.
 * update progress logs in remotedaemon send/receive
-* Resuming an interrupted download does not jump the progress to reflect
-  the amount already present. And, it fails at the end.
+* Resuming an interrupted download fails at the end.
 * Think about locking some more. What happens if the connection to the peer
   is dropped while we think we're locking content there from being dropped?
 * merge to master

From 99c36f318c7174c05f04579fa76e2b7c478dba6c Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 7 Dec 2016 15:06:07 -0400
Subject: [PATCH 199/367] open file for append, not write, so resuming works

WriteMode zeros any existing content, so the seek filled with zeros, and
verification failed after download.
---
 P2P/Annex.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/P2P/Annex.hs b/P2P/Annex.hs
index 4105abe32e..daab74447a 100644
--- a/P2P/Annex.hs
+++ b/P2P/Annex.hs
@@ -120,7 +120,7 @@ runLocal runmode runner a = case a of
 		-- a client.
 		Client -> ta
 	storefile dest (Offset o) (Len l) b = liftIO $ do
-		withBinaryFile dest WriteMode $ \h -> do
+		withBinaryFile dest AppendMode $ \h -> do
 			when (o /= 0) $
 				hSeek h AbsoluteSeek o
 			L.hPut h b

From 76b46afed18c802c97dba952975f72f5f15cd1e2 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 7 Dec 2016 15:21:38 -0400
Subject: [PATCH 200/367] add section on tor speed

---
 doc/tips/peer_to_peer_network_with_tor.mdwn | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/doc/tips/peer_to_peer_network_with_tor.mdwn b/doc/tips/peer_to_peer_network_with_tor.mdwn
index 718a9218d3..3e6a4bd59c 100644
--- a/doc/tips/peer_to_peer_network_with_tor.mdwn
+++ b/doc/tips/peer_to_peer_network_with_tor.mdwn
@@ -63,7 +63,7 @@ so peer1 is a remote of peer2 and peer2 is a remote of peer1.
 
 Any number of peers can be connected this way, within reason.
 
-## git-annex remotedaemon
+## starting git-annex remotedaemon
 
 Notice the `git annex remotedaemon` being run in the above examples.
 That command runs the Tor hidden service so that other peers
@@ -77,6 +77,20 @@ You can do that with a simple cron job:
 If you use the git-annex assistant, and have it auto-starting on boot, it
 will take care of starting the remotedaemon for you.
 
+## speed of large transfers
+
+Tor prioritizes security over speed, and the Tor network only has so much
+bandwidth to go around. So, distributing large quantities (gigabytes) 
+of data over Tor may be slow, and should probably be avoided.
+
+One way to avoid sending much data over tor is to set up an encrypted
+[[special_remote|special_remotes]]. git-annex knows that Tor is rather
+expensive to use, so if a file is available on a special remote as well as
+over Tor, it will download it from the special remote.
+
+You can contribute to the Tor network by
+[running a Tor relay or bridge](https://www.torproject.org/getinvolved/relays.html.en).
+
 ## onion addresses and authentication
 
 You don't need to know about this, but it might be helpful to understand

From db79b69aa0dcf785c830d81ba8849e3ad4224a1b Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 7 Dec 2016 15:24:28 -0400
Subject: [PATCH 201/367] ReadWriteMode not AppendMode

AppendMode does not allow seeking..
---
 P2P/Annex.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/P2P/Annex.hs b/P2P/Annex.hs
index daab74447a..1ef54de1d8 100644
--- a/P2P/Annex.hs
+++ b/P2P/Annex.hs
@@ -120,7 +120,7 @@ runLocal runmode runner a = case a of
 		-- a client.
 		Client -> ta
 	storefile dest (Offset o) (Len l) b = liftIO $ do
-		withBinaryFile dest AppendMode $ \h -> do
+		withBinaryFile dest ReadWriteMode $ \h -> do
 			when (o /= 0) $
 				hSeek h AbsoluteSeek o
 			L.hPut h b

From 0541f19beaeb9f1759a96d6251c67f307b402263 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 7 Dec 2016 15:36:39 -0400
Subject: [PATCH 202/367] fix math error that caused resumes to always fail

---
 P2P/Annex.hs      | 2 +-
 doc/todo/tor.mdwn | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/P2P/Annex.hs b/P2P/Annex.hs
index 1ef54de1d8..7e07038d3d 100644
--- a/P2P/Annex.hs
+++ b/P2P/Annex.hs
@@ -125,4 +125,4 @@ runLocal runmode runner a = case a of
 				hSeek h AbsoluteSeek o
 			L.hPut h b
 		sz <- getFileSize dest
-		return (toInteger sz == l)
+		return (toInteger sz == l + o)
diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index af4fa19939..fece15bebe 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -6,8 +6,9 @@ Current todo list:
 
 * copy --to peer seems to make the remotedaemon buffer the content in
   memory, more than I'd expect.
+* There are no error messages when things fail. Need to convert P2P runner
+  from Maybe to Either String.
 * update progress logs in remotedaemon send/receive
-* Resuming an interrupted download fails at the end.
 * Think about locking some more. What happens if the connection to the peer
   is dropped while we think we're locking content there from being dropped?
 * merge to master

From e5476d42ce53dfe99bdc514ac8bb9b8d28ab3b53 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 7 Dec 2016 15:38:37 -0400
Subject: [PATCH 203/367] update

---
 doc/todo/tor.mdwn | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index fece15bebe..a06d1aabf3 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -5,13 +5,12 @@ Mostly working!
 Current todo list:
 
 * copy --to peer seems to make the remotedaemon buffer the content in
-  memory, more than I'd expect.
+  memory, more than I'd expect. Check if this is a memory leak..
 * There are no error messages when things fail. Need to convert P2P runner
   from Maybe to Either String.
 * update progress logs in remotedaemon send/receive
 * Think about locking some more. What happens if the connection to the peer
   is dropped while we think we're locking content there from being dropped?
-* merge to master
 
 Eventually:
 

From 46cbd65808ac951fbde06ce298ef6f126eec8859 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 7 Dec 2016 15:44:18 -0400
Subject: [PATCH 204/367] add page for tor special remote

---
 doc/special_remotes.mdwn     |  1 +
 doc/special_remotes/tor.mdwn | 10 ++++++++++
 2 files changed, 11 insertions(+)
 create mode 100644 doc/special_remotes/tor.mdwn

diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index 2d99069be0..1dc3d8705d 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -18,6 +18,7 @@ They cannot be used by other git commands though.
 * [[tahoe]]
 * [[web]]
 * [[bittorrent]]
+* [[tor]]
 * [[xmpp]]
 * [[hook]]
 
diff --git a/doc/special_remotes/tor.mdwn b/doc/special_remotes/tor.mdwn
new file mode 100644
index 0000000000..12d3dfedf6
--- /dev/null
+++ b/doc/special_remotes/tor.mdwn
@@ -0,0 +1,10 @@
+git-annex can communicate over the Tor network. This allows direct
+communication between git-annex repositories, no matter where they are
+located.
+
+A git remote using tor has an url that looks like
+`tor-annex::2lssjzicvsxkdc2v.onion:19984`
+
+To set this up, use [[git-annex-enabletor]] and [[git-annex-p2p]],
+and run [[git-annex-remotedaemon]] to serve the Tor hidden service.
+It's explained in detail in [[tips/peer_to_peer_network_with_tor]].

From 138f618002cbc54f7451672d00e582a1e281a699 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 7 Dec 2016 15:49:15 -0400
Subject: [PATCH 205/367] todo

---
 doc/todo/tor.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index a06d1aabf3..5a8c880391 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -6,6 +6,8 @@ Current todo list:
 
 * copy --to peer seems to make the remotedaemon buffer the content in
   memory, more than I'd expect. Check if this is a memory leak..
+* Resuming an interrupted transfer fails at the end, despite having gotten
+  the whole correct file content.
 * There are no error messages when things fail. Need to convert P2P runner
   from Maybe to Either String.
 * update progress logs in remotedaemon send/receive

From ed6a5bc271b5e8e60f4c55cb78bdea8b46dbb972 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 7 Dec 2016 15:50:54 -0400
Subject: [PATCH 206/367] devblog

---
 doc/devblog/day_434__it_works.mdwn | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 doc/devblog/day_434__it_works.mdwn

diff --git a/doc/devblog/day_434__it_works.mdwn b/doc/devblog/day_434__it_works.mdwn
new file mode 100644
index 0000000000..5ab81d422e
--- /dev/null
+++ b/doc/devblog/day_434__it_works.mdwn
@@ -0,0 +1,19 @@
+Git annex transfers over Tor worked correctly the first time I tried them
+today. I had been expecting protocol implementation bugs, so this was a
+nice surprise!
+
+Of course there were some bugs to fix. I had forgotten to add UUID
+discovery to `git annex p2p --link`. And, resuming interrupted transfers
+was buggy.
+
+Spent some time adding progress updates to the Tor remote. I was curious to
+see what speed transfers would run. Speed will of course vary depending on
+the Tor relays being used, but this example with a 100 mb file is not bad:
+
+	copy big4 (to peer1...) 
+	62%          1.5MB/s 24s
+
+There are still a couple of [[known bugs|todo/tor]], 
+but I've merged the `tor` branch into `master` already.
+
+Today's work was sponsored by Ethan Aubin.

From 62043df289e2f60c7b3d63491f63fc83e2111163 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 7 Dec 2016 16:00:12 -0400
Subject: [PATCH 207/367] git-annex-metadata-gui yay!

---
 doc/devblog/day_434__it_works.mdwn                        | 8 ++++++++
 doc/related_software.mdwn                                 | 4 ++++
 .../comment_1_1ce311d8328ea370a6a3494adea0f5db._comment   | 8 ++++++++
 3 files changed, 20 insertions(+)
 create mode 100644 doc/tips/a_gui_for_metadata_operations/comment_1_1ce311d8328ea370a6a3494adea0f5db._comment

diff --git a/doc/devblog/day_434__it_works.mdwn b/doc/devblog/day_434__it_works.mdwn
index 5ab81d422e..75d096b31c 100644
--- a/doc/devblog/day_434__it_works.mdwn
+++ b/doc/devblog/day_434__it_works.mdwn
@@ -16,4 +16,12 @@ the Tor relays being used, but this example with a 100 mb file is not bad:
 There are still a couple of [[known bugs|todo/tor]], 
 but I've merged the `tor` branch into `master` already.
 
+----
+
+Alpernebbi has built a GUI for editing git-annex metadata.
+Something I always wanted!  
+[[Read about it here|tips/a_gui_for_metadata_operations]]
+
+----
+
 Today's work was sponsored by Ethan Aubin.
diff --git a/doc/related_software.mdwn b/doc/related_software.mdwn
index bfc68186c2..3f099063e5 100644
--- a/doc/related_software.mdwn
+++ b/doc/related_software.mdwn
@@ -16,6 +16,10 @@ designed to interoperate with it.
 * [Magit](http://github.com/magit/magit), an Emacs mode for Git, has
   [an extension](https://github.com/magit/magit-annex) for git annex.
 
+* [git-annex-metadata-gui](https://github.com/alpernebbi/git-annex-metadata-gui)
+  is a GUI for editing the metadata. An easier alternative to using
+  [[git-annex-metadata]] at the command line.
+
 * [DataLad](http://datalad.org/) uses git-annex to provide access to
   scientific data available from various sources.
 
diff --git a/doc/tips/a_gui_for_metadata_operations/comment_1_1ce311d8328ea370a6a3494adea0f5db._comment b/doc/tips/a_gui_for_metadata_operations/comment_1_1ce311d8328ea370a6a3494adea0f5db._comment
new file mode 100644
index 0000000000..2a55de0beb
--- /dev/null
+++ b/doc/tips/a_gui_for_metadata_operations/comment_1_1ce311d8328ea370a6a3494adea0f5db._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-12-07T19:58:11Z"
+ content="""
+Thank you for this, I've always wanted such a GUI, and it's been a common
+user request!
+"""]]

From 8e5f8d91c5c7416aae1cc40a3e0a26015ac6fb00 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 7 Dec 2016 16:06:56 -0400
Subject: [PATCH 208/367] add git-annex-adaptor

---
 doc/related_software.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/related_software.mdwn b/doc/related_software.mdwn
index 3f099063e5..acd9df8380 100644
--- a/doc/related_software.mdwn
+++ b/doc/related_software.mdwn
@@ -46,4 +46,7 @@ designed to interoperate with it.
 * [git-annex-watcher](https://github.com/rubiojr/git-annex-watcher)
   is a status icon for your desktop.
 
+* [git-annex-adaptor](https://github.com/alpernebbi/git-annex-adapter)
+  is a python interface to git-annex.
+
 See also [[not]] for software that is *not* related to git-annex, but similar.

From 0766d032bf62244e6e3ad888afd0082eb60f8308 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 7 Dec 2016 16:09:30 -0400
Subject: [PATCH 209/367] link to git-annex-metadata-gui

---
 doc/metadata.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/metadata.mdwn b/doc/metadata.mdwn
index f1f0ceab70..3b61a47d52 100644
--- a/doc/metadata.mdwn
+++ b/doc/metadata.mdwn
@@ -13,6 +13,8 @@ Some of the things you can do with metadata include:
   For example `git annex find --metadata tag=foo --or --metadata tag=bar`
 * Using it in [[preferred_content]] expressions. 
   For example "metadata=tag=important or not metadata=author=me"
+* Editing and viewing it with git-annex-metadata-gui,
+  [[tips/a_gui_for_metadata_operations]].
 
 Each file (actually the underlying key) can have any number of metadata
 fields, which each can have any number of values. For example, to tag

From 44c55e2441484e77ad09f9c0a907ad8694f1de89 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 7 Dec 2016 16:11:30 -0400
Subject: [PATCH 210/367] fix links

---
 doc/tips/peer_to_peer_network_with_tor.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/tips/peer_to_peer_network_with_tor.mdwn b/doc/tips/peer_to_peer_network_with_tor.mdwn
index 3e6a4bd59c..11d977cca0 100644
--- a/doc/tips/peer_to_peer_network_with_tor.mdwn
+++ b/doc/tips/peer_to_peer_network_with_tor.mdwn
@@ -1,12 +1,12 @@
 git-annex has recently gotten support for running as a
-[Tor](http://http://torproject.org/) hidden service. This is a nice secure
+[Tor](https://torproject.org/) hidden service. This is a nice secure
 and easy to use way to connect repositories between peers in different
 locations, without needing any central server.
 
 ## setting up the first peer
 
 First, you need to get Tor installed and running. See
-[their website](http://http://torproject.org/), or try a command like:
+[their website](https://torproject.org/), or try a command like:
 
 	sudo apt-get install tor
 

From df67626cb7b19bd5f8278b23b9c11962661ccab1 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 8 Dec 2016 13:58:03 -0400
Subject: [PATCH 211/367] fix build with old ghc

---
 P2P/Protocol.hs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/P2P/Protocol.hs b/P2P/Protocol.hs
index f51ea7f98c..8aabb37d7d 100644
--- a/P2P/Protocol.hs
+++ b/P2P/Protocol.hs
@@ -29,6 +29,8 @@ import System.Exit (ExitCode(..))
 import System.IO
 import qualified Data.ByteString.Lazy as L
 import Data.Char
+import Control.Applicative
+import Prelude
 
 newtype Offset = Offset Integer
 	deriving (Show)

From b613ccf7a330e5eb3f5e245ae4c6cd857708016d Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 8 Dec 2016 13:58:11 -0400
Subject: [PATCH 212/367] update

---
 doc/todo/tor.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index 5a8c880391..fa078ac6b2 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -16,6 +16,9 @@ Current todo list:
 
 Eventually:
 
+* Limiting authtokens to read-only access.
+* Revoking authtokens. (This and read-only need a name associated with an
+  authtoken, so the user can adjust its configuration after creating it.)
 * address exchange via electrum-mnemonic or magic wormhole (see PAKE)
 * webapp UI for easy pairing
 * friend-of-a-friend peer discovery to build more interconnected networks

From e56506d83c636b130a8663811d70eae33ffed604 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 8 Dec 2016 14:14:08 -0400
Subject: [PATCH 213/367] include error message when unable to connect to peer

---
 Command/P2P.hs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Command/P2P.hs b/Command/P2P.hs
index 21632f3da5..db69eff973 100644
--- a/Command/P2P.hs
+++ b/Command/P2P.hs
@@ -81,7 +81,7 @@ linkRemote remotename = do
 	setup (P2PAddressAuth addr authtoken) = do
 		g <- Annex.gitRepo
 		conn <- liftIO $ connectPeer g addr
-			`catchNonAsync` giveup "Unable to connect with peer. Please check that the peer is connected to the network, and try again."
+			`catchNonAsync` connerror
 		u <- getUUID
 		v <- liftIO $ runNetProto conn $ P2P.auth u authtoken
 		case v of
@@ -96,3 +96,4 @@ linkRemote remotename = do
 					storeP2PRemoteAuthToken addr authtoken
 				return ok
 			_ -> giveup "Unable to authenticate with peer. Please check the address and try again."
+	connerror e = giveup $ "Unable to connect with peer. Please check that the peer is connected to the network, and try again. ("  ++ show e ++ ")"

From c05f4eb6318ce0a783253c05fcbf3420392255f3 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 8 Dec 2016 15:15:29 -0400
Subject: [PATCH 214/367] fix laziness problem in git relaying

The switch to hGetMetered subtly changed the laziness of how DATA was
read, and broke git protocol relaying. Fix by sending received data to
the git process's stdin immediately, which ensures that the lazy
bytestring is all read from the peer before going on to process the next
message from the peer.
---
 P2P/IO.hs | 40 ++++++++++++++++++++--------------------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/P2P/IO.hs b/P2P/IO.hs
index ea15ecfc3a..b8e34333b6 100644
--- a/P2P/IO.hs
+++ b/P2P/IO.hs
@@ -167,7 +167,7 @@ runRelay runner (RelayHandle hout) (RelayHandle hin) = bracket setup cleanup go
   where
 	setup = do
 		v <- newEmptyMVar
-		void $ async $ relayFeeder runner v
+		void $ async $ relayFeeder runner v hin
 		void $ async $ relayReader v hout
 		return v
 	
@@ -175,7 +175,7 @@ runRelay runner (RelayHandle hout) (RelayHandle hin) = bracket setup cleanup go
 		hClose hin
 		hClose hout
 	
-	go v = relayHelper runner v hin
+	go v = relayHelper runner v
 
 runRelayService :: P2PConnection -> RunProto IO -> Service -> IO (Maybe ())
 runRelayService conn runner service = bracket setup cleanup go
@@ -195,7 +195,7 @@ runRelayService conn runner service = bracket setup cleanup go
 			, std_in = CreatePipe
 			}
 		v <- newEmptyMVar
-		void $ async $ relayFeeder runner v
+		void $ async $ relayFeeder runner v hin
 		void $ async $ relayReader v hout
 		waiter <- async $ waitexit v pid
 		return (v, waiter, hin, hout, pid)
@@ -206,8 +206,8 @@ runRelayService conn runner service = bracket setup cleanup go
 		cancel waiter
 		void $ waitForProcess pid
 
-	go (v, _, hin, _, _) = do
-		r <- relayHelper runner v hin
+	go (v, _, _, _, _) = do
+		r <- relayHelper runner v
 		case r of
 			Nothing -> return Nothing
 			Just exitcode -> runner $ net $ relayToPeer (RelayDone exitcode)
@@ -215,16 +215,12 @@ runRelayService conn runner service = bracket setup cleanup go
 	waitexit v pid = putMVar v . RelayDone =<< waitForProcess pid
 
 -- Processes RelayData as it is put into the MVar.
-relayHelper :: RunProto IO -> MVar RelayData -> Handle -> IO (Maybe ExitCode)
-relayHelper runner v hin = loop
+relayHelper :: RunProto IO -> MVar RelayData -> IO (Maybe ExitCode)
+relayHelper runner v = loop
   where
 	loop = do
 		d <- takeMVar v
 		case d of
-			RelayFromPeer b -> do
-				L.hPut hin b
-				hFlush hin
-				loop
 			RelayToPeer b -> do
 				r <- runner $ net $ relayToPeer (RelayToPeer b)
 				case r of
@@ -233,21 +229,25 @@ relayHelper runner v hin = loop
 			RelayDone exitcode -> do
 				_ <- runner $ net $ relayToPeer (RelayDone exitcode)
 				return (Just exitcode)
+			RelayFromPeer _ -> loop -- not handled here
 
--- Takes input from the peer, and puts it into the MVar for processing.
+-- Takes input from the peer, and sends it to the relay process's stdin.
 -- Repeats until the peer tells it it's done or hangs up.
-relayFeeder :: RunProto IO -> MVar RelayData -> IO ()
-relayFeeder runner v = loop
+relayFeeder :: RunProto IO -> MVar RelayData -> Handle -> IO ()
+relayFeeder runner v hin = loop
   where
 	loop = do
 		mrd <- runner $ net relayFromPeer
 		case mrd of
-			Nothing -> putMVar v (RelayDone (ExitFailure 1))
-			Just rd -> do
-				putMVar v rd
-				case rd of
-					RelayDone _ -> return ()
-					_ -> loop
+			Nothing ->
+				putMVar v (RelayDone (ExitFailure 1))
+			Just (RelayDone exitcode) ->
+				putMVar v (RelayDone exitcode)
+			Just (RelayFromPeer b) -> do
+				L.hPut hin b
+				hFlush hin
+				loop
+			Just (RelayToPeer _) -> loop -- not handled here
 
 -- Reads input from the Handle and puts it into the MVar for relaying to
 -- the peer. Continues until EOF on the Handle.

From af41519126f4b841250f14a849aab375c48a1891 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 8 Dec 2016 15:47:49 -0400
Subject: [PATCH 215/367] convert P2P runners from Maybe to Either String

So we get some useful error messages when things fail.

This commit was sponsored by Peter Hogg on Patreon.
---
 CmdLine/GitRemoteTorAnnex.hs  |  6 ++--
 Command/P2P.hs                |  5 +--
 P2P/Annex.hs                  | 17 +++++-----
 P2P/IO.hs                     | 59 +++++++++++++++++++----------------
 Remote/P2P.hs                 | 25 +++++++++------
 RemoteDaemon/Transport/Tor.hs |  9 ++++--
 doc/todo/tor.mdwn             |  2 --
 7 files changed, 69 insertions(+), 54 deletions(-)

diff --git a/CmdLine/GitRemoteTorAnnex.hs b/CmdLine/GitRemoteTorAnnex.hs
index c4bf26c859..5208a47ca3 100644
--- a/CmdLine/GitRemoteTorAnnex.hs
+++ b/CmdLine/GitRemoteTorAnnex.hs
@@ -34,8 +34,8 @@ run (_remotename:address:[]) = forever $ do
 		| otherwise = parseAddressPort address
 	go service = do
 		ready
-		res <- connectService onionaddress onionport service 
-		exitWith (fromMaybe (ExitFailure 1) res)
+		either giveup exitWith
+			=<< connectService onionaddress onionport service
 	ready = do
 		putStrLn ""
 		hFlush stdout
@@ -50,7 +50,7 @@ parseAddressPort s =
 		Nothing -> giveup "onion address must include port number"
 		Just p -> (OnionAddress a, p)
 
-connectService :: OnionAddress -> OnionPort -> Service -> IO (Maybe ExitCode)
+connectService :: OnionAddress -> OnionPort -> Service -> IO (Either String ExitCode)
 connectService address port service = do
 	state <- Annex.new =<< Git.CurrentRepo.get
 	Annex.eval state $ do
diff --git a/Command/P2P.hs b/Command/P2P.hs
index db69eff973..ea4dd7c656 100644
--- a/Command/P2P.hs
+++ b/Command/P2P.hs
@@ -85,7 +85,7 @@ linkRemote remotename = do
 		u <- getUUID
 		v <- liftIO $ runNetProto conn $ P2P.auth u authtoken
 		case v of
-			Just (Just theiruuid) -> do
+			Right (Just theiruuid) -> do
 				ok <- inRepo $ Git.Command.runBool
 					[ Param "remote", Param "add"
 					, Param remotename
@@ -95,5 +95,6 @@ linkRemote remotename = do
 					storeUUIDIn (remoteConfig remotename "uuid") theiruuid
 					storeP2PRemoteAuthToken addr authtoken
 				return ok
-			_ -> giveup "Unable to authenticate with peer. Please check the address and try again."
+			Right Nothing -> giveup "Unable to authenticate with peer. Please check the address and try again."
+			Left e -> giveup $ "Unable to authenticate with peer: " ++ e
 	connerror e = giveup $ "Unable to connect with peer. Please check that the peer is connected to the network, and try again. ("  ++ show e ++ ")"
diff --git a/P2P/Annex.hs b/P2P/Annex.hs
index 7e07038d3d..d55d69bdbc 100644
--- a/P2P/Annex.hs
+++ b/P2P/Annex.hs
@@ -31,15 +31,15 @@ data RunMode
 	| Client
 
 -- Full interpreter for Proto, that can receive and send objects.
-runFullProto :: RunMode -> P2PConnection -> Proto a -> Annex (Maybe a)
+runFullProto :: RunMode -> P2PConnection -> Proto a -> Annex (Either String a)
 runFullProto runmode conn = go
   where
 	go :: RunProto Annex
-	go (Pure v) = pure (Just v)
+	go (Pure v) = pure (Right v)
 	go (Free (Net n)) = runNet conn go n
 	go (Free (Local l)) = runLocal runmode go l
 
-runLocal :: RunMode -> RunProto Annex -> LocalF (Proto a) -> Annex (Maybe a)
+runLocal :: RunMode -> RunProto Annex -> LocalF (Proto a) -> Annex (Either String a)
 runLocal runmode runner a = case a of
 	TmpContentSize k next -> do
 		tmp <- fromRepo $ gitAnnexTmpObjectLocation k
@@ -68,9 +68,10 @@ runLocal runmode runner a = case a of
 							hSeek h AbsoluteSeek o
 						L.hGetContents h
 				case v' of
-					Left _ -> return Nothing
+					Left e -> return (Left (show e))
 					Right b -> runner (next b)
-			_ -> return Nothing
+			Right Nothing -> return (Left "content not available")
+			Left e -> return (Left (show e))
 	StoreContent k af o l b next -> do
 		ok <- flip catchNonAsync (const $ return False) $
 			transfer download k af $
@@ -84,12 +85,12 @@ runLocal runmode runner a = case a of
 	SetPresent k u next -> do
 		v <- tryNonAsync $ logChange k u InfoPresent
 		case v of
-			Left _ -> return Nothing
+			Left e -> return (Left (show e))
 			Right () -> runner next
 	CheckContentPresent k next -> do
 		v <- tryNonAsync $ inAnnex k
 		case v of
-			Left _ -> return Nothing
+			Left e -> return (Left (show e))
 			Right result -> runner (next result)
 	RemoveContent k next -> do
 		v <- tryNonAsync $ lockContentForRemoval k $ \contentlock -> do
@@ -97,7 +98,7 @@ runLocal runmode runner a = case a of
 				logStatus k InfoMissing
 				return True
 		case v of
-			Left _ -> return Nothing
+			Left e -> return (Left (show e))
 			Right result -> runner (next result)
 	TryLockContent k protoaction next -> do
 		v <- tryNonAsync $ lockContentShared k $ \verifiedcopy -> 
diff --git a/P2P/IO.hs b/P2P/IO.hs
index b8e34333b6..a2af5946c0 100644
--- a/P2P/IO.hs
+++ b/P2P/IO.hs
@@ -42,7 +42,7 @@ import qualified Data.ByteString as B
 import qualified Data.ByteString.Lazy as L
 
 -- Type of interpreters of the Proto free monad.
-type RunProto m = forall a. (MonadIO m, MonadMask m) => Proto a -> m (Maybe a)
+type RunProto m = forall a. (MonadIO m, MonadMask m) => Proto a -> m (Either String a)
 
 data P2PConnection = P2PConnection
 	{ connRepo :: Repo
@@ -80,31 +80,31 @@ setupHandle s = do
 -- This only runs Net actions. No Local actions will be run
 -- (those need the Annex monad) -- if the interpreter reaches any,
 -- it returns Nothing.
-runNetProto :: P2PConnection -> Proto a -> IO (Maybe a)
+runNetProto :: P2PConnection -> Proto a -> IO (Either String a)
 runNetProto conn = go
   where
 	go :: RunProto IO
-	go (Pure v) = pure (Just v)
+	go (Pure v) = pure (Right v)
 	go (Free (Net n)) = runNet conn go n
-	go (Free (Local _)) = return Nothing
+	go (Free (Local _)) = return (Left "unexpected annex operation attempted")
 
 -- Interpreter of the Net part of Proto.
 --
 -- An interpreter of Proto has to be provided, to handle the rest of Proto
 -- actions.
-runNet :: (MonadIO m, MonadMask m) => P2PConnection -> RunProto m -> NetF (Proto a) -> m (Maybe a)
+runNet :: (MonadIO m, MonadMask m) => P2PConnection -> RunProto m -> NetF (Proto a) -> m (Either String a)
 runNet conn runner f = case f of
 	SendMessage m next -> do
 		v <- liftIO $ tryNonAsync $ do
 			hPutStrLn (connOhdl conn) (unwords (formatMessage m))
 			hFlush (connOhdl conn)
 		case v of
-			Left _e -> return Nothing
+			Left e -> return (Left (show e))
 			Right () -> runner next
 	ReceiveMessage next -> do
 		v <- liftIO $ tryNonAsync $ hGetLine (connIhdl conn)
 		case v of
-			Left _e -> return Nothing
+			Left e -> return (Left (show e))
 			Right l -> case parseMessage l of
 				Just m -> runner (next m)
 				Nothing -> runner $ do
@@ -118,11 +118,12 @@ runNet conn runner f = case f of
 			return ok
 		case v of
 			Right True -> runner next
-			_ -> return Nothing
+			Right False -> return (Left "short data write")
+			Left e -> return (Left (show e))
 	ReceiveBytes len p next -> do
 		v <- liftIO $ tryNonAsync $ receiveExactly len (connIhdl conn) p
 		case v of
-			Left _e -> return Nothing
+			Left e -> return (Left (show e))
 			Right b -> runner (next b)
 	CheckAuthToken _u t next -> do
 		let authed = connCheckAuth conn t
@@ -130,13 +131,13 @@ runNet conn runner f = case f of
 	Relay hin hout next -> do
 		v <- liftIO $ runRelay runnerio hin hout
 		case v of
-			Nothing -> return Nothing
-			Just exitcode -> runner (next exitcode)
+			Left e -> return (Left e)
+			Right exitcode -> runner (next exitcode)
 	RelayService service next -> do
 		v <- liftIO $ runRelayService conn runnerio service
 		case v of
-			Nothing -> return Nothing
-			Just () -> runner next
+			Left e -> return (Left e)
+			Right () -> runner next
   where
 	-- This is only used for running Net actions when relaying,
 	-- so it's ok to use runNetProto, despite it not supporting
@@ -162,8 +163,10 @@ sendExactly (Len n) b h p = do
 receiveExactly :: Len -> Handle -> MeterUpdate -> IO L.ByteString
 receiveExactly (Len n) h p = hGetMetered h (Just n) p
 
-runRelay :: RunProto IO -> RelayHandle -> RelayHandle -> IO (Maybe ExitCode)
-runRelay runner (RelayHandle hout) (RelayHandle hin) = bracket setup cleanup go
+runRelay :: RunProto IO -> RelayHandle -> RelayHandle -> IO (Either String ExitCode)
+runRelay runner (RelayHandle hout) (RelayHandle hin) = 
+	bracket setup cleanup go
+		`catchNonAsync` (return . Left . show)
   where
 	setup = do
 		v <- newEmptyMVar
@@ -177,8 +180,10 @@ runRelay runner (RelayHandle hout) (RelayHandle hin) = bracket setup cleanup go
 	
 	go v = relayHelper runner v
 
-runRelayService :: P2PConnection -> RunProto IO -> Service -> IO (Maybe ())
-runRelayService conn runner service = bracket setup cleanup go
+runRelayService :: P2PConnection -> RunProto IO -> Service -> IO (Either String ())
+runRelayService conn runner service = 
+	bracket setup cleanup go
+		`catchNonAsync` (return . Left . show)
   where
 	cmd = case service of
 		UploadPack -> "upload-pack"
@@ -209,13 +214,13 @@ runRelayService conn runner service = bracket setup cleanup go
 	go (v, _, _, _, _) = do
 		r <- relayHelper runner v
 		case r of
-			Nothing -> return Nothing
-			Just exitcode -> runner $ net $ relayToPeer (RelayDone exitcode)
+			Left e -> return (Left (show e))
+			Right exitcode -> runner $ net $ relayToPeer (RelayDone exitcode)
 	
 	waitexit v pid = putMVar v . RelayDone =<< waitForProcess pid
 
 -- Processes RelayData as it is put into the MVar.
-relayHelper :: RunProto IO -> MVar RelayData -> IO (Maybe ExitCode)
+relayHelper :: RunProto IO -> MVar RelayData -> IO (Either String ExitCode)
 relayHelper runner v = loop
   where
 	loop = do
@@ -224,11 +229,11 @@ relayHelper runner v = loop
 			RelayToPeer b -> do
 				r <- runner $ net $ relayToPeer (RelayToPeer b)
 				case r of
-					Nothing -> return Nothing
-					Just () -> loop
+					Left e -> return (Left e)
+					Right () -> loop
 			RelayDone exitcode -> do
 				_ <- runner $ net $ relayToPeer (RelayDone exitcode)
-				return (Just exitcode)
+				return (Right exitcode)
 			RelayFromPeer _ -> loop -- not handled here
 
 -- Takes input from the peer, and sends it to the relay process's stdin.
@@ -239,15 +244,15 @@ relayFeeder runner v hin = loop
 	loop = do
 		mrd <- runner $ net relayFromPeer
 		case mrd of
-			Nothing ->
+			Left _e ->
 				putMVar v (RelayDone (ExitFailure 1))
-			Just (RelayDone exitcode) ->
+			Right (RelayDone exitcode) ->
 				putMVar v (RelayDone exitcode)
-			Just (RelayFromPeer b) -> do
+			Right (RelayFromPeer b) -> do
 				L.hPut hin b
 				hFlush hin
 				loop
-			Just (RelayToPeer _) -> loop -- not handled here
+			Right (RelayToPeer _) -> loop -- not handled here
 
 -- Reads input from the Handle and puts it into the MVar for relaying to
 -- the peer. Continues until EOF on the Handle.
diff --git a/Remote/P2P.hs b/Remote/P2P.hs
index 1d7ede30ff..bc0e2f9230 100644
--- a/Remote/P2P.hs
+++ b/Remote/P2P.hs
@@ -127,14 +127,15 @@ runProto u addr connpool a = withConnection u addr connpool (runProto' a)
 runProto' :: P2P.Proto a -> Connection -> Annex (Connection, Maybe a)
 runProto' _ ClosedConnection = return (ClosedConnection, Nothing)
 runProto' a (OpenConnection conn) = do
-	r <- runFullProto Client conn a
+	v <- runFullProto Client conn a
 	-- When runFullProto fails, the connection is no longer usable,
 	-- so close it.
-	if isJust r
-		then return (OpenConnection conn, r)
-		else do
+	case v of
+		Left e -> do
+			warning e
 			liftIO $ closeConnection conn
-			return (ClosedConnection, r)
+			return (ClosedConnection, Nothing)
+		Right r -> return (OpenConnection conn, Just r)
 
 -- Uses an open connection if one is available in the ConnectionPool;
 -- otherwise opens a new connection.
@@ -176,16 +177,20 @@ openConnection u addr = do
 			res <- liftIO $ runNetProto conn $
 				P2P.auth myuuid authtoken
 			case res of
-				Just (Just theiruuid)
+				Right (Just theiruuid)
 					| u == theiruuid -> return (OpenConnection conn)
 					| otherwise -> do
 						liftIO $ closeConnection conn
 						warning "Remote peer uuid seems to have changed."
 						return ClosedConnection
-				_ -> do
-					liftIO $ closeConnection conn
+				Right Nothing -> do
 					warning "Unable to authenticate with peer."
+					liftIO $ closeConnection conn
 					return ClosedConnection
-		Left _e -> do
-			warning "Unable to connect to peer."
+				Left e -> do
+					warning e
+					liftIO $ closeConnection conn
+					return ClosedConnection
+		Left e -> do
+			warning $ "Unable to connect to peer. (" ++ show e ++ ")"
 			return ClosedConnection
diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs
index e5d4e97add..ab794a77ef 100644
--- a/RemoteDaemon/Transport/Tor.hs
+++ b/RemoteDaemon/Transport/Tor.hs
@@ -92,10 +92,15 @@ serveClient th u r q = bracket setup cleanup go
 				}
 			v <- liftIO $ runNetProto conn $ serveAuth u
 			case v of
-				Just (Just theiruuid) -> void $ 
+				Right (Just theiruuid) -> void $ 
 					runFullProto (Serving theiruuid) conn $
 						serveAuthed u
-				_ -> return ()
+				Right Nothing -> do
+					liftIO $ debugM "remotedaemon" "TOR connection failed to authenticate"
+					return ()
+				Left e -> do
+					warning e
+					return ()
 		-- Merge the duplicated state back in.
 		liftAnnex th $ mergeState st'
 		debugM "remotedaemon" "done with TOR connection"
diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index fa078ac6b2..12ef7561f1 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -8,8 +8,6 @@ Current todo list:
   memory, more than I'd expect. Check if this is a memory leak..
 * Resuming an interrupted transfer fails at the end, despite having gotten
   the whole correct file content.
-* There are no error messages when things fail. Need to convert P2P runner
-  from Maybe to Either String.
 * update progress logs in remotedaemon send/receive
 * Think about locking some more. What happens if the connection to the peer
   is dropped while we think we're locking content there from being dropped?

From 23438fb80f02fde6d2184149d0fa9e547f786e0a Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 8 Dec 2016 15:56:36 -0400
Subject: [PATCH 216/367] move to debug

---
 RemoteDaemon/Transport/Tor.hs | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs
index ab794a77ef..514d40006d 100644
--- a/RemoteDaemon/Transport/Tor.hs
+++ b/RemoteDaemon/Transport/Tor.hs
@@ -95,12 +95,10 @@ serveClient th u r q = bracket setup cleanup go
 				Right (Just theiruuid) -> void $ 
 					runFullProto (Serving theiruuid) conn $
 						serveAuthed u
-				Right Nothing -> do
-					liftIO $ debugM "remotedaemon" "TOR connection failed to authenticate"
-					return ()
-				Left e -> do
-					warning e
-					return ()
+				Right Nothing -> liftIO $
+					debugM "remotedaemon" "TOR connection failed to authenticate"
+				Left e -> liftIO $
+					debugM "remotedaemon" ("Error while serving TOR connection: " ++ e)
 		-- Merge the duplicated state back in.
 		liftAnnex th $ mergeState st'
 		debugM "remotedaemon" "done with TOR connection"

From 39b7eb73506836416efb3d1b5ba95c894506d025 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 8 Dec 2016 16:00:29 -0400
Subject: [PATCH 217/367] update

---
 doc/todo/tor.mdwn | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index 12ef7561f1..b262c793c6 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -6,8 +6,10 @@ Current todo list:
 
 * copy --to peer seems to make the remotedaemon buffer the content in
   memory, more than I'd expect. Check if this is a memory leak..
-* Resuming an interrupted transfer fails at the end, despite having gotten
-  the whole correct file content.
+* When a transfer can't be done because another transfer of the same 
+  object is already in progress, the message about this is output by the
+  remotedaemon --debug, but not forwarded to the peer, which shows
+  "Connection reset by peer"
 * update progress logs in remotedaemon send/receive
 * Think about locking some more. What happens if the connection to the peer
   is dropped while we think we're locking content there from being dropped?

From c6972cb91440649ed568a4dda53ca14aa86eada6 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 8 Dec 2016 16:02:26 -0400
Subject: [PATCH 218/367] better format error

---
 Remote/P2P.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Remote/P2P.hs b/Remote/P2P.hs
index bc0e2f9230..0813098f18 100644
--- a/Remote/P2P.hs
+++ b/Remote/P2P.hs
@@ -188,7 +188,7 @@ openConnection u addr = do
 					liftIO $ closeConnection conn
 					return ClosedConnection
 				Left e -> do
-					warning e
+					warning $ "Problem communicating with peer. (" ++ e ++ ")"
 					liftIO $ closeConnection conn
 					return ClosedConnection
 		Left e -> do

From 2ad06ded7e9f11e681594950c12535ea08d42eee Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 8 Dec 2016 16:28:07 -0400
Subject: [PATCH 219/367] force sofar calculation

This could avoid a memory leak. It would only happen when
the meter didn't look at sofar.
---
 Utility/Metered.hs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Utility/Metered.hs b/Utility/Metered.hs
index b80d3ae3f9..e21e18cf14 100644
--- a/Utility/Metered.hs
+++ b/Utility/Metered.hs
@@ -5,7 +5,7 @@
  - License: BSD-2-clause
  -}
 
-{-# LANGUAGE TypeSynonymInstances #-}
+{-# LANGUAGE TypeSynonymInstances, BangPatterns #-}
 
 module Utility.Metered where
 
@@ -93,7 +93,7 @@ meteredWrite' meterupdate h = go zeroBytesProcessed . L.toChunks
 	go sofar [] = return sofar
 	go sofar (c:cs) = do
 		S.hPut h c
-		let sofar' = addBytesProcessed sofar $ S.length c
+		let !sofar' = addBytesProcessed sofar $ S.length c
 		meterupdate sofar'
 		go sofar' cs
 
@@ -138,7 +138,7 @@ hGetMetered h wantsize meterupdate = lazyRead zeroBytesProcessed
 				hClose h
 				return $ L.empty
 			else do
-				let sofar' = addBytesProcessed sofar (S.length c)
+				let !sofar' = addBytesProcessed sofar (S.length c)
 				meterupdate sofar'
 				if keepgoing (fromBytesProcessed sofar')
 					then do

From d8f1a0d95c12765f0abea573d8fc7a4875808655 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 8 Dec 2016 16:31:08 -0400
Subject: [PATCH 220/367] move byteable to main dep list

Only the webapp had pulled it in, but the authtoken code uses it now.
---
 git-annex.cabal | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index c894e66104..465769ea0e 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -370,6 +370,7 @@ Executable git-annex
    feed,
    regex-tdfa,
    socks,
+   byteable,
    securemem
   CC-Options: -Wall
   GHC-Options: -Wall -fno-warn-tabs
@@ -473,8 +474,7 @@ Executable git-annex
      crypto-api,
      clientsession,
      template-haskell,
-     shakespeare (>= 2.0.0),
-     byteable
+     shakespeare (>= 2.0.0)
     CPP-Options: -DWITH_WEBAPP
 
   if flag(Pairing)

From b8fea9f08de66f462abf753b808fa214d8407d32 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 8 Dec 2016 16:35:33 -0400
Subject: [PATCH 221/367] we have a memory leak

---
 doc/todo/tor.mdwn | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index b262c793c6..ca3e8f19a5 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -4,8 +4,13 @@ Mostly working!
 
 Current todo list:
 
-* copy --to peer seems to make the remotedaemon buffer the content in
-  memory, more than I'd expect. Check if this is a memory leak..
+* copy --to peer of a 100 mb file causes the memory of the remotedaemon
+  to creep up from 40 mb to 136 mb. Once the transfer is done, the
+  remotedaemon continues using all that memory. Memory leak. Profile it.  
+  (The sending process creeps up some initially, but stops at 45 mb used.
+  That could just be buffering.)  
+  (copy --from peer does not leak on either end; the remotedaemon uses 34
+  mb and the receiver 44 mb.)
 * When a transfer can't be done because another transfer of the same 
   object is already in progress, the message about this is output by the
   remotedaemon --debug, but not forwarded to the peer, which shows

From 0288e934cc827f673273c1202c72ab06b5c3c5e5 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 8 Dec 2016 16:55:40 -0400
Subject: [PATCH 222/367] update instructions to work better

---
 doc/ekg.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/ekg.mdwn b/doc/ekg.mdwn
index 508fd2e929..26d0128a63 100644
--- a/doc/ekg.mdwn
+++ b/doc/ekg.mdwn
@@ -20,11 +20,11 @@ git-annex will continue to work.
 For the really tricky memory leaks, here's how to make a profiling build of
 git-annex.
 
-1. `cabal configure` with only the flags you really need
-2. `cabal build --ghc-options="-prof -auto-all -caf-all"`
+1. `cabal configure --enable-profiling`  
   This will probably fail due to some missing profiling libraries.
   You have to get the profiling versions of all needed haskell libraries
   installed somehow.
+2. `cabal build`
 3. Run git-annex with the special flags `+RTS -hc -p`
 4. Reproduce the memory leak problem.
 5. If the assistant was run, stop it.

From 43e7044b43136da7ebf3d6cb511749c34c3781ec Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 8 Dec 2016 17:10:24 -0400
Subject: [PATCH 223/367] comment

---
 Command/FromKey.hs                            |  4 ++-
 ..._37643180ecbc6c6bb0504b3acb18d1e7._comment | 31 +++++++++++++++++++
 2 files changed, 34 insertions(+), 1 deletion(-)
 create mode 100644 doc/bugs/git-annex_fromkey_barfs_on_utf-8_input/comment_2_37643180ecbc6c6bb0504b3acb18d1e7._comment

diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index dca63aabe2..6a81c1f74c 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -45,7 +45,9 @@ startMass = do
 	next massAdd
 
 massAdd :: CommandPerform
-massAdd = go True =<< map (separate (== ' ')) . lines <$> liftIO getContents
+massAdd = do
+	liftIO $ fileEncoding stdin
+	go True =<< map (separate (== ' ')) . lines <$> liftIO getContents
   where
 	go status [] = next $ return status
 	go status ((keyname,f):rest) | not (null keyname) && not (null f) = do
diff --git a/doc/bugs/git-annex_fromkey_barfs_on_utf-8_input/comment_2_37643180ecbc6c6bb0504b3acb18d1e7._comment b/doc/bugs/git-annex_fromkey_barfs_on_utf-8_input/comment_2_37643180ecbc6c6bb0504b3acb18d1e7._comment
new file mode 100644
index 0000000000..0b2cabdf9d
--- /dev/null
+++ b/doc/bugs/git-annex_fromkey_barfs_on_utf-8_input/comment_2_37643180ecbc6c6bb0504b3acb18d1e7._comment
@@ -0,0 +1,31 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-12-08T21:03:14Z"
+ content="""
+I'm not able to reproduce the original reported problem; it works even when
+using the C locale. However, it may be that this patch would fix it:
+
+<pre>
+diff --git a/Command/FromKey.hs b/Command/FromKey.hs
+index dca63aa..6a81c1f 100644
+--- a/Command/FromKey.hs
++++ b/Command/FromKey.hs
+@@ -45,7 +45,9 @@ startMass = do
+ 	next massAdd
+ 
+ massAdd :: CommandPerform
+-massAdd = go True =<< map (separate (== ' ')) . lines <$> liftIO getContents
++massAdd = do
++	liftIO $ fileEncoding stdin
++	go True =<< map (separate (== ' ')) . lines <$> liftIO getContents
+   where
+ 	go status [] = next $ return status
+ 	go status ((keyname,f):rest) | not (null keyname) && not (null f) = do
+</pre>
+
+(The NeuroDebian git-annex-standalone may well have had its locale
+installation broken by [[!commit c07981122672f6cc87ca08efb57d8a7b1e2f5725]],
+which assumes that the git-annex.linux is writable by the user.
+I doubt that is related to the original bug report.)
+"""]]

From 8e00efb938545538a60a303a19288865c8b032da Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 8 Dec 2016 17:10:48 -0400
Subject: [PATCH 224/367] didn't mean to commit this change yet

---
 Command/FromKey.hs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index 6a81c1f74c..dca63aabe2 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -45,9 +45,7 @@ startMass = do
 	next massAdd
 
 massAdd :: CommandPerform
-massAdd = do
-	liftIO $ fileEncoding stdin
-	go True =<< map (separate (== ' ')) . lines <$> liftIO getContents
+massAdd = go True =<< map (separate (== ' ')) . lines <$> liftIO getContents
   where
 	go status [] = next $ return status
 	go status ((keyname,f):rest) | not (null keyname) && not (null f) = do

From da8c65601f5d572c6851a492d5ef3946e9d8ac34 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 8 Dec 2016 17:12:02 -0400
Subject: [PATCH 225/367] comment

---
 .../comment_1_14818629616e3daeb8293b710298ce31._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/ssh___34__Include__34___feature_broke_Android/comment_1_14818629616e3daeb8293b710298ce31._comment

diff --git a/doc/bugs/ssh___34__Include__34___feature_broke_Android/comment_1_14818629616e3daeb8293b710298ce31._comment b/doc/bugs/ssh___34__Include__34___feature_broke_Android/comment_1_14818629616e3daeb8293b710298ce31._comment
new file mode 100644
index 0000000000..0cf33b8b32
--- /dev/null
+++ b/doc/bugs/ssh___34__Include__34___feature_broke_Android/comment_1_14818629616e3daeb8293b710298ce31._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-12-08T21:11:31Z"
+ content="""
+Indeed, the ssh bundled in the apk is shockingly old by now, and needs to
+get updated.
+"""]]

From 41ed0770df9c4d802f32fe305daa3fd5c091a4a6 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 8 Dec 2016 17:17:01 -0400
Subject: [PATCH 226/367] consistent caps

---
 RemoteDaemon/Transport/Tor.hs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs
index 514d40006d..7afc9c3333 100644
--- a/RemoteDaemon/Transport/Tor.hs
+++ b/RemoteDaemon/Transport/Tor.hs
@@ -51,7 +51,7 @@ server th@(TransportHandle (LocalRepo r) _) = do
 	modifyFileMode sock $ addModes
 		[groupReadMode, groupWriteMode, otherReadMode, otherWriteMode]
 	listen soc 2
-	debugM "remotedaemon" "tor hidden service running"
+	debugM "remotedaemon" "Tor hidden service running"
 	forever $ do
 		(conn, _) <- accept soc
 		h <- setupHandle conn
@@ -63,7 +63,7 @@ server th@(TransportHandle (LocalRepo r) _) = do
 			)
 		unless ok $ do
 			hClose h
-			warningIO "dropped TOR connection, too busy"
+			warningIO "dropped Tor connection, too busy"
 
 -- How many clients to serve at a time, maximum. This is to avoid DOS
 -- attacks.
@@ -76,7 +76,7 @@ serveClient th u r q = bracket setup cleanup go
 	setup = atomically $ readTBQueue q
 	cleanup = hClose
 	go h = do
-		debugM "remotedaemon" "serving a TOR connection"
+		debugM "remotedaemon" "serving a Tor connection"
 		-- Avoid doing any work in the liftAnnex, since only one
 		-- can run at a time.
 		st <- liftAnnex th dupState
@@ -96,9 +96,9 @@ serveClient th u r q = bracket setup cleanup go
 					runFullProto (Serving theiruuid) conn $
 						serveAuthed u
 				Right Nothing -> liftIO $
-					debugM "remotedaemon" "TOR connection failed to authenticate"
+					debugM "remotedaemon" "Tor connection failed to authenticate"
 				Left e -> liftIO $
-					debugM "remotedaemon" ("Error while serving TOR connection: " ++ e)
+					debugM "remotedaemon" ("Error while serving Tor connection: " ++ e)
 		-- Merge the duplicated state back in.
 		liftAnnex th $ mergeState st'
-		debugM "remotedaemon" "done with TOR connection"
+		debugM "remotedaemon" "done with Tor connection"

From 5654085e7ab6cb68411f4f8c070208658d9cb48a Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 8 Dec 2016 17:34:26 -0400
Subject: [PATCH 227/367] profiling

---
 doc/todo/tor.mdwn | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index ca3e8f19a5..3a6de57a52 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -11,6 +11,18 @@ Current todo list:
   That could just be buffering.)  
   (copy --from peer does not leak on either end; the remotedaemon uses 34
   mb and the receiver 44 mb.)
+
+  Profiling results: Leak is in hGetMetered, or perhaps in
+  the consumer of the data it reads. Graph shows `ARR_WORDS` is
+  the type; that must be a bytestring.
+
+<pre>                                                                                                                   individual      inherited
+COST CENTRE                                               MODULE                               no.      entries  %time %alloc   %time %alloc
+                     tryNonAsync                          Utility.Exception                    3241          0    0.0    0.1    49.3   65.8
+                      receiveExactly                      P2P.IO                               3429          0    0.0    0.0    49.3   65.7
+                       hGetMetered                        Utility.Metered                      3430          0   49.1   65.6    49.3   65.7
+</pre>
+
 * When a transfer can't be done because another transfer of the same 
   object is already in progress, the message about this is output by the
   remotedaemon --debug, but not forwarded to the peer, which shows

From 3b9f6c1b2bdde07325ff4e238edf8f624777449c Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 8 Dec 2016 17:50:59 -0400
Subject: [PATCH 228/367] analysis

---
 doc/todo/tor.mdwn | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index 3a6de57a52..39ffb0ceb9 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -23,6 +23,13 @@ COST CENTRE                                               MODULE
                        hGetMetered                        Utility.Metered                      3430          0   49.1   65.6    49.3   65.7
 </pre>
 
+  Switching to L.hGet, it still leaks, so seems hGetMetered is not at fault
+  and the bytestring is being buffered excessively somehow before it
+  reaches the file.
+
+  Aha! While a send is in progress like this, .git/annex/tmp is empty!
+  The whole file is being buffered in memory and written at the end.
+
 * When a transfer can't be done because another transfer of the same 
   object is already in progress, the message about this is output by the
   remotedaemon --debug, but not forwarded to the peer, which shows

From 095593a9afd68a22d12363cd5873a3bc89a17063 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 8 Dec 2016 18:25:51 -0400
Subject: [PATCH 229/367] correction

---
 doc/todo/tor.mdwn | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index 39ffb0ceb9..a4700fd5d0 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -24,11 +24,7 @@ COST CENTRE                                               MODULE
 </pre>
 
   Switching to L.hGet, it still leaks, so seems hGetMetered is not at fault
-  and the bytestring is being buffered excessively somehow before it
-  reaches the file.
-
-  Aha! While a send is in progress like this, .git/annex/tmp is empty!
-  The whole file is being buffered in memory and written at the end.
+  and the bytestring is being buffered excessively somehow.
 
 * When a transfer can't be done because another transfer of the same 
   object is already in progress, the message about this is output by the

From 0f4ee4f298c9e09ca3fd9c66d131234f923edfa8 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 8 Dec 2016 18:26:03 -0400
Subject: [PATCH 230/367] fix memory leak

I'm unsure why this fixed it, but it did. Seems to suggest that the
memory leak is not due to a bug in my code, but that ghc didn't manage
to take full advantage of laziness, or was failing to gc something it
could have.
---
 P2P/Annex.hs      | 26 +++++++++++++++-----------
 P2P/Protocol.hs   | 10 +++++-----
 doc/todo/tor.mdwn | 22 ----------------------
 3 files changed, 20 insertions(+), 38 deletions(-)

diff --git a/P2P/Annex.hs b/P2P/Annex.hs
index d55d69bdbc..3b95c8c021 100644
--- a/P2P/Annex.hs
+++ b/P2P/Annex.hs
@@ -72,15 +72,15 @@ runLocal runmode runner a = case a of
 					Right b -> runner (next b)
 			Right Nothing -> return (Left "content not available")
 			Left e -> return (Left (show e))
-	StoreContent k af o l b next -> do
+	StoreContent k af o l getb next -> do
 		ok <- flip catchNonAsync (const $ return False) $
 			transfer download k af $
 				getViaTmp AlwaysVerify k $ \tmp ->
-					unVerified $ storefile tmp o l b
+					unVerified $ storefile tmp o l getb
 		runner (next ok)
-	StoreContentTo dest o l b next -> do
+	StoreContentTo dest o l getb next -> do
 		ok <- flip catchNonAsync (const $ return False) $
-			storefile dest o l b
+			storefile dest o l getb
 		runner (next ok)
 	SetPresent k u next -> do
 		v <- tryNonAsync $ logChange k u InfoPresent
@@ -120,10 +120,14 @@ runLocal runmode runner a = case a of
 		-- Transfer logs are updated higher in the stack when
 		-- a client.
 		Client -> ta
-	storefile dest (Offset o) (Len l) b = liftIO $ do
-		withBinaryFile dest ReadWriteMode $ \h -> do
-			when (o /= 0) $
-				hSeek h AbsoluteSeek o
-			L.hPut h b
-		sz <- getFileSize dest
-		return (toInteger sz == l + o)
+	storefile dest (Offset o) (Len l) getb = do
+		v <- runner getb
+		case v of
+			Right b -> liftIO $ do
+				withBinaryFile dest ReadWriteMode $ \h -> do
+					when (o /= 0) $
+						hSeek h AbsoluteSeek o
+					L.hPut h b
+				sz <- liftIO $ getFileSize dest
+				return (toInteger sz == l + o)
+			Left e -> error e
diff --git a/P2P/Protocol.hs b/P2P/Protocol.hs
index 8aabb37d7d..5d6c6fcc54 100644
--- a/P2P/Protocol.hs
+++ b/P2P/Protocol.hs
@@ -200,7 +200,7 @@ data LocalF c
 	| ReadContent Key AssociatedFile Offset (L.ByteString -> c)
 	-- ^ Lazily reads the content of a key. Note that the content
 	-- may change while it's being sent.
-	| StoreContent Key AssociatedFile Offset Len L.ByteString (Bool -> c)
+	| StoreContent Key AssociatedFile Offset Len (Proto L.ByteString) (Bool -> c)
 	-- ^ Stores content to the key's temp file starting at an offset.
 	-- Once the whole content of the key has been stored, moves the
 	-- temp file into place as the content of the key, and returns True.
@@ -208,7 +208,7 @@ data LocalF c
 	-- Note: The ByteString may not contain the entire remaining content
 	-- of the key. Only once the temp file size == Len has the whole
 	-- content been transferred.
-	| StoreContentTo FilePath Offset Len L.ByteString (Bool -> c)
+	| StoreContentTo FilePath Offset Len (Proto L.ByteString) (Bool -> c)
 	-- ^ Stores the content to a temp file starting at an offset.
 	-- Once the whole content of the key has been stored, returns True.
 	--
@@ -388,7 +388,7 @@ sendContent key af offset@(Offset n) p = do
 	net $ sendBytes len content p'
 	checkSuccess
 
-receiveContent :: MeterUpdate -> Local Len -> (Offset -> Len -> L.ByteString -> Local Bool) -> (Offset -> Message) -> Proto Bool
+receiveContent :: MeterUpdate -> Local Len -> (Offset -> Len -> Proto L.ByteString -> Local Bool) -> (Offset -> Message) -> Proto Bool
 receiveContent p sizer storer mkmsg = do
 	Len n <- local sizer
 	let p' = offsetMeterUpdate p (toBytesProcessed n)
@@ -397,8 +397,8 @@ receiveContent p sizer storer mkmsg = do
 	r <- net receiveMessage
 	case r of
 		DATA len -> do
-			ok <- local . storer offset len
-				=<< net (receiveBytes len p')
+			ok <- local $ storer offset len
+				(net (receiveBytes len p'))
 			sendSuccess ok
 			return ok
 		_ -> do
diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index a4700fd5d0..79822ab19e 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -4,28 +4,6 @@ Mostly working!
 
 Current todo list:
 
-* copy --to peer of a 100 mb file causes the memory of the remotedaemon
-  to creep up from 40 mb to 136 mb. Once the transfer is done, the
-  remotedaemon continues using all that memory. Memory leak. Profile it.  
-  (The sending process creeps up some initially, but stops at 45 mb used.
-  That could just be buffering.)  
-  (copy --from peer does not leak on either end; the remotedaemon uses 34
-  mb and the receiver 44 mb.)
-
-  Profiling results: Leak is in hGetMetered, or perhaps in
-  the consumer of the data it reads. Graph shows `ARR_WORDS` is
-  the type; that must be a bytestring.
-
-<pre>                                                                                                                   individual      inherited
-COST CENTRE                                               MODULE                               no.      entries  %time %alloc   %time %alloc
-                     tryNonAsync                          Utility.Exception                    3241          0    0.0    0.1    49.3   65.8
-                      receiveExactly                      P2P.IO                               3429          0    0.0    0.0    49.3   65.7
-                       hGetMetered                        Utility.Metered                      3430          0   49.1   65.6    49.3   65.7
-</pre>
-
-  Switching to L.hGet, it still leaks, so seems hGetMetered is not at fault
-  and the bytestring is being buffered excessively somehow.
-
 * When a transfer can't be done because another transfer of the same 
   object is already in progress, the message about this is output by the
   remotedaemon --debug, but not forwarded to the peer, which shows

From 1f3ed1b6b240dba88c07e754e62fc04483ba7a1b Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 8 Dec 2016 18:42:59 -0400
Subject: [PATCH 231/367] avoid remotedaemon outputting misc Messages when
 running

It's a daemon or is communictaing with the assistant over stdio, so
should not display checksum messages etc while serving the P2P protocol
etc.
---
 RemoteDaemon/Core.hs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/RemoteDaemon/Core.hs b/RemoteDaemon/Core.hs
index 446948da6d..2166c2b7a2 100644
--- a/RemoteDaemon/Core.hs
+++ b/RemoteDaemon/Core.hs
@@ -20,6 +20,7 @@ import Utility.SimpleProtocol
 import Utility.ThreadScheduler
 import Config
 import Annex.Ssh
+import Types.Messages
 
 import Control.Concurrent
 import Control.Concurrent.Async
@@ -151,7 +152,9 @@ genTransportHandle :: IO TransportHandle
 genTransportHandle = do
 	annexstate <- newMVar =<< Annex.new =<< Git.CurrentRepo.get
 	g <- Annex.repo <$> readMVar annexstate
-	return $ TransportHandle (LocalRepo g) annexstate
+	let h = TransportHandle (LocalRepo g) annexstate
+	liftAnnex h $ Annex.setOutput QuietOutput
+	return h
 
 updateTransportHandle :: TransportHandle -> IO TransportHandle
 updateTransportHandle h@(TransportHandle _g annexstate) = do

From 38516b2fcac4894bd78ae5d63b350e298accb4bb Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 8 Dec 2016 19:56:02 -0400
Subject: [PATCH 232/367] update progress logs in remotedaemon send/receive

---
 Annex/Transfer.hs |  5 +++++
 P2P/Annex.hs      | 57 ++++++++++++++++++++++++++++-------------------
 P2P/Protocol.hs   | 43 +++++++++++++++--------------------
 doc/todo/tor.mdwn |  1 -
 4 files changed, 57 insertions(+), 49 deletions(-)

diff --git a/Annex/Transfer.hs b/Annex/Transfer.hs
index 323600e96a..b33dace4af 100644
--- a/Annex/Transfer.hs
+++ b/Annex/Transfer.hs
@@ -45,6 +45,11 @@ instance Observable (Bool, Verification) where
 	observeBool = fst
 	observeFailure = (False, UnVerified)
 
+instance Observable (Either e Bool) where
+	observeBool (Left _) = False
+	observeBool (Right b) = b
+	observeFailure = Right False
+
 upload :: Observable v => UUID -> Key -> AssociatedFile -> RetryDecider -> (MeterUpdate -> Annex v) -> NotifyWitness -> Annex v
 upload u key f d a _witness = guardHaveUUID u $ 
 	runTransfer (Transfer Upload u key) f d a
diff --git a/P2P/Annex.hs b/P2P/Annex.hs
index 3b95c8c021..93da9e69bc 100644
--- a/P2P/Annex.hs
+++ b/P2P/Annex.hs
@@ -20,9 +20,9 @@ import P2P.Protocol
 import P2P.IO
 import Logs.Location
 import Types.NumCopies
+import Utility.Metered
 
 import Control.Monad.Free
-import qualified Data.ByteString.Lazy as L
 
 -- When we're serving a peer, we know their uuid, and can use it to update
 -- transfer logs.
@@ -52,35 +52,33 @@ runLocal runmode runner a = case a of
 		let getsize = liftIO . catchMaybeIO . getFileSize
 		size <- inAnnex' isJust Nothing getsize k
 		runner (next (Len <$> size))
-	-- TODO transfer log not updated
-	ReadContent k af (Offset o) next -> do
+	ReadContent k af o sender next -> do
 		v <- tryNonAsync $ prepSendAnnex k
 		case v of
-			-- The check can detect a problem after the
-			-- content is sent, but we don't use it.
-			-- Instead, the receiving peer must AlwaysVerify
-			-- the content it receives.
+			-- The check can detect if the file
+			-- changed while it was transferred, but we don't
+			-- use it. Instead, the receiving peer must
+			-- AlwaysVerify the content it receives.
 			Right (Just (f, _check)) -> do
-				v' <- tryNonAsync $ -- transfer upload k af $
-					liftIO $ do
-						h <- openBinaryFile f ReadMode
-						when (o /= 0) $
-							hSeek h AbsoluteSeek o
-						L.hGetContents h
+				v' <- tryNonAsync $
+					transfer upload k af $
+						sinkfile f o sender
 				case v' of
 					Left e -> return (Left (show e))
-					Right b -> runner (next b)
-			Right Nothing -> return (Left "content not available")
+					Right (Left e) -> return (Left (show e))
+					Right (Right ok) -> runner (next ok)
+			-- content not available
+ 			Right Nothing -> runner (next False)
 			Left e -> return (Left (show e))
 	StoreContent k af o l getb next -> do
 		ok <- flip catchNonAsync (const $ return False) $
-			transfer download k af $
+			transfer download k af $ \p ->
 				getViaTmp AlwaysVerify k $ \tmp ->
-					unVerified $ storefile tmp o l getb
+					unVerified $ storefile tmp o l getb p
 		runner (next ok)
 	StoreContentTo dest o l getb next -> do
 		ok <- flip catchNonAsync (const $ return False) $
-			storefile dest o l getb
+			storefile dest o l getb nullMeterUpdate
 		runner (next ok)
 	SetPresent k u next -> do
 		v <- tryNonAsync $ logChange k u InfoPresent
@@ -116,18 +114,31 @@ runLocal runmode runner a = case a of
 	transfer mk k af ta = case runmode of
 		-- Update transfer logs when serving.
 		Serving theiruuid -> 
-			mk theiruuid k af noRetry (const ta) noNotification
+			mk theiruuid k af noRetry ta noNotification
 		-- Transfer logs are updated higher in the stack when
 		-- a client.
-		Client -> ta
-	storefile dest (Offset o) (Len l) getb = do
+		Client -> ta nullMeterUpdate
+	
+	storefile dest (Offset o) (Len l) getb p = do
+		let p' = offsetMeterUpdate p (toBytesProcessed o)
 		v <- runner getb
 		case v of
 			Right b -> liftIO $ do
 				withBinaryFile dest ReadWriteMode $ \h -> do
 					when (o /= 0) $
 						hSeek h AbsoluteSeek o
-					L.hPut h b
-				sz <- liftIO $ getFileSize dest
+					meteredWrite p' h b
+				sz <- getFileSize dest
 				return (toInteger sz == l + o)
 			Left e -> error e
+	
+	sinkfile f (Offset o) sender p = bracket setup cleanup go
+	  where
+		setup = liftIO $ openBinaryFile f ReadMode
+		cleanup = liftIO . hClose
+		go h = do
+			let p' = offsetMeterUpdate p (toBytesProcessed o)
+			when (o /= 0) $
+				liftIO $ hSeek h AbsoluteSeek o
+			b <- liftIO $ hGetContentsMetered h p'
+			runner (sender b)
diff --git a/P2P/Protocol.hs b/P2P/Protocol.hs
index 5d6c6fcc54..03c7c70cf7 100644
--- a/P2P/Protocol.hs
+++ b/P2P/Protocol.hs
@@ -197,9 +197,10 @@ data LocalF c
 	| ContentSize Key (Maybe Len -> c)
 	-- ^ Gets size of the content of a key, when the full content is
 	-- present.
-	| ReadContent Key AssociatedFile Offset (L.ByteString -> c)
-	-- ^ Lazily reads the content of a key. Note that the content
-	-- may change while it's being sent.
+	| ReadContent Key AssociatedFile Offset (L.ByteString -> Proto Bool) (Bool -> c)
+	-- ^ Reads the content of a key and sends it to the callback.
+	-- Note that the content may change while it's being sent.
+	-- If the content is not available, sends L.empty to the callback.
 	| StoreContent Key AssociatedFile Offset Len (Proto L.ByteString) (Bool -> c)
 	-- ^ Stores content to the key's temp file starting at an offset.
 	-- Once the whole content of the key has been stored, moves the
@@ -381,12 +382,20 @@ serveAuthed myuuid = void $ serverLoop handler
 	handler _ = return ServerUnexpected
 
 sendContent :: Key -> AssociatedFile -> Offset -> MeterUpdate -> Proto Bool
-sendContent key af offset@(Offset n) p = do
-	let p' = offsetMeterUpdate p (toBytesProcessed n)
-	(len, content) <- readContentLen key af offset
-	net $ sendMessage (DATA len)
-	net $ sendBytes len content p'
-	checkSuccess
+sendContent key af offset@(Offset n) p = go =<< local (contentSize key)
+  where
+ 	go Nothing = sender (Len 0) L.empty
+	go (Just (Len totallen)) = do
+		let len = totallen - n
+		if len <= 0
+			then sender (Len 0) L.empty
+			else local $ readContent key af offset $
+				sender (Len len)
+	sender len content = do
+		let p' = offsetMeterUpdate p (toBytesProcessed n)
+		net $ sendMessage (DATA len)
+		net $ sendBytes len content p'
+		checkSuccess
 
 receiveContent :: MeterUpdate -> Local Len -> (Offset -> Len -> Proto L.ByteString -> Local Bool) -> (Offset -> Message) -> Proto Bool
 receiveContent p sizer storer mkmsg = do
@@ -419,22 +428,6 @@ sendSuccess :: Bool -> Proto ()
 sendSuccess True = net $ sendMessage SUCCESS
 sendSuccess False = net $ sendMessage FAILURE
 
--- Reads content from an offset. The Len should correspond to
--- the length of the ByteString, but to avoid buffering the content
--- in memory, is gotten using contentSize.
-readContentLen :: Key -> AssociatedFile -> Offset -> Proto (Len, L.ByteString)
-readContentLen key af (Offset offset) = go =<< local (contentSize key)
-  where
-	go Nothing = return (Len 0, L.empty)
-	go (Just (Len totallen)) = do
-		let len = totallen - offset
-		if len <= 0
-			then return (Len 0, L.empty)
-			else do
-				content <- local $
-					readContent key af (Offset offset)
-				return (Len len, content)
-
 connect :: Service -> Handle -> Handle -> Proto ExitCode
 connect service hin hout = do
 	net $ sendMessage (CONNECT service)
diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index 79822ab19e..01fa2635ac 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -8,7 +8,6 @@ Current todo list:
   object is already in progress, the message about this is output by the
   remotedaemon --debug, but not forwarded to the peer, which shows
   "Connection reset by peer"
-* update progress logs in remotedaemon send/receive
 * Think about locking some more. What happens if the connection to the peer
   is dropped while we think we're locking content there from being dropped?
 

From e84165dc5e6acd6469d8cb6975a533d5b632cd0a Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 8 Dec 2016 20:15:12 -0400
Subject: [PATCH 233/367] more todo

---
 doc/todo/tor.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index 01fa2635ac..00ded54c58 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -8,6 +8,9 @@ Current todo list:
   object is already in progress, the message about this is output by the
   remotedaemon --debug, but not forwarded to the peer, which shows
   "Connection reset by peer"
+* git annex drop --from peer1, when the peer does not have a file,
+  shows "<socket: 18>: hGetLine: end of file"; the peer has closed the
+  connection.
 * Think about locking some more. What happens if the connection to the peer
   is dropped while we think we're locking content there from being dropped?
 

From 9c5c24aa0a682c77252c86d1d91f67b3e143bff3 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 8 Dec 2016 20:55:46 -0400
Subject: [PATCH 234/367] todo

---
 doc/todo/tor.mdwn | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index 00ded54c58..ce8d8e98ae 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -4,6 +4,10 @@ Mostly working!
 
 Current todo list:
 
+* Current use of hGetLine to read protocol messages allows memory DOS by
+  sending a very long line. May also have line ending problems across OS's. 
+  Switch to instead reading a packed data structure that starts with its
+  length, and refuse to read messages > 32k.
 * When a transfer can't be done because another transfer of the same 
   object is already in progress, the message about this is output by the
   remotedaemon --debug, but not forwarded to the peer, which shows

From ca1bcdcd7c028ed4391d372fcde6677a10d5e7ea Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 9 Dec 2016 12:35:45 -0400
Subject: [PATCH 235/367] improve warning on connection loss

---
 Remote/P2P.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Remote/P2P.hs b/Remote/P2P.hs
index 0813098f18..f0848f8318 100644
--- a/Remote/P2P.hs
+++ b/Remote/P2P.hs
@@ -132,7 +132,7 @@ runProto' a (OpenConnection conn) = do
 	-- so close it.
 	case v of
 		Left e -> do
-			warning e
+			warning $ "Lost connection to peer (" ++ e ++ ")"
 			liftIO $ closeConnection conn
 			return (ClosedConnection, Nothing)
 		Right r -> return (OpenConnection conn, Just r)

From 71e8cd408edb576221e6bcc90333949a755857b3 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 9 Dec 2016 12:47:57 -0400
Subject: [PATCH 236/367] content removal is supposed to succed if the content
 was already not present

---
 P2P/Annex.hs      | 12 ++++++++----
 doc/todo/tor.mdwn |  3 ---
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/P2P/Annex.hs b/P2P/Annex.hs
index 93da9e69bc..771a72126b 100644
--- a/P2P/Annex.hs
+++ b/P2P/Annex.hs
@@ -91,10 +91,14 @@ runLocal runmode runner a = case a of
 			Left e -> return (Left (show e))
 			Right result -> runner (next result)
 	RemoveContent k next -> do
-		v <- tryNonAsync $ lockContentForRemoval k $ \contentlock -> do
-				removeAnnex contentlock
-				logStatus k InfoMissing
-				return True
+		v <- tryNonAsync $
+			ifM (Annex.Content.inAnnex key)
+				( lockContentForRemoval k $ \contentlock -> do
+					removeAnnex contentlock
+					logStatus k InfoMissing
+					return True
+				, return True
+				)
 		case v of
 			Left e -> return (Left (show e))
 			Right result -> runner (next result)
diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index ce8d8e98ae..11d441daba 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -12,9 +12,6 @@ Current todo list:
   object is already in progress, the message about this is output by the
   remotedaemon --debug, but not forwarded to the peer, which shows
   "Connection reset by peer"
-* git annex drop --from peer1, when the peer does not have a file,
-  shows "<socket: 18>: hGetLine: end of file"; the peer has closed the
-  connection.
 * Think about locking some more. What happens if the connection to the peer
   is dropped while we think we're locking content there from being dropped?
 

From bdf2a31424eaf57b053ff554f9a3154eb13ef2fe Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 9 Dec 2016 12:54:12 -0400
Subject: [PATCH 237/367] typo

---
 P2P/Annex.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/P2P/Annex.hs b/P2P/Annex.hs
index 771a72126b..d24e65b0fc 100644
--- a/P2P/Annex.hs
+++ b/P2P/Annex.hs
@@ -92,7 +92,7 @@ runLocal runmode runner a = case a of
 			Right result -> runner (next result)
 	RemoveContent k next -> do
 		v <- tryNonAsync $
-			ifM (Annex.Content.inAnnex key)
+			ifM (Annex.Content.inAnnex k)
 				( lockContentForRemoval k $ \contentlock -> do
 					removeAnnex contentlock
 					logStatus k InfoMissing

From 0f3a3ff1e531a5c670339d6c87553f0e28a7ff8c Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 9 Dec 2016 12:54:18 -0400
Subject: [PATCH 238/367] make clear that log is only updated after successful
 removal

This does not change behavior, because an exception is thrown on
unsuccessful removal. But is clearer.
---
 Remote/Git.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index 41fb46e82e..9602f81c69 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -356,9 +356,9 @@ dropKey r key
 			commitOnCleanup r $ onLocal r $ do
 				ensureInitialized
 				whenM (Annex.Content.inAnnex key) $ do
-					Annex.Content.lockContentForRemoval key
+					Annex.Content.lockContentForRemoval key $ do
 						Annex.Content.removeAnnex
-					logStatus key InfoMissing
+						logStatus key InfoMissing
 					Annex.Content.saveState True
 				return True
 	| Git.repoIsHttp (repo r) = giveup "dropping from http remote not supported"

From 58f5d41cacd1d7fd0ee59e4454421e41fb2527c8 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 9 Dec 2016 12:56:38 -0400
Subject: [PATCH 239/367] fix

---
 Remote/Git.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index 9602f81c69..37a9f48b3e 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -356,8 +356,8 @@ dropKey r key
 			commitOnCleanup r $ onLocal r $ do
 				ensureInitialized
 				whenM (Annex.Content.inAnnex key) $ do
-					Annex.Content.lockContentForRemoval key $ do
-						Annex.Content.removeAnnex
+					Annex.Content.lockContentForRemoval key $ \lock -> do
+						Annex.Content.removeAnnex lock
 						logStatus key InfoMissing
 					Annex.Content.saveState True
 				return True

From 3d759a03224951fbaabd38ebbbf476db320e7eb1 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 9 Dec 2016 13:00:19 -0400
Subject: [PATCH 240/367] debug on error serving peer

---
 RemoteDaemon/Transport/Tor.hs | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs
index 7afc9c3333..0dd1d1ba27 100644
--- a/RemoteDaemon/Transport/Tor.hs
+++ b/RemoteDaemon/Transport/Tor.hs
@@ -92,13 +92,16 @@ serveClient th u r q = bracket setup cleanup go
 				}
 			v <- liftIO $ runNetProto conn $ serveAuth u
 			case v of
-				Right (Just theiruuid) -> void $ 
-					runFullProto (Serving theiruuid) conn $
+				Right (Just theiruuid) -> void $ do
+					v' <- runFullProto (Serving theiruuid) conn $
 						serveAuthed u
+					case v' of
+						Right () -> return ()
+						Left e -> liftIO $ debugM "remotedaemon" ("Tor connection error: " ++ e)
 				Right Nothing -> liftIO $
 					debugM "remotedaemon" "Tor connection failed to authenticate"
 				Left e -> liftIO $
-					debugM "remotedaemon" ("Error while serving Tor connection: " ++ e)
+					debugM "remotedaemon" ("Tor connection error before authentication: " ++ e)
 		-- Merge the duplicated state back in.
 		liftAnnex th $ mergeState st'
 		debugM "remotedaemon" "done with Tor connection"

From 15be5c04a6900f6068fda0d4b3fa972af2b38219 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 9 Dec 2016 13:34:00 -0400
Subject: [PATCH 241/367] git-annex-shell, remotedaemon, git remote: Fix some
 memory DOS attacks.

The attacker could just send a very lot of data, with no \n and it would
all be buffered in memory until the kernel killed git-annex or perhaps OOM
killed some other more valuable process.

This is a low impact security hole, only affecting communication between
local git-annex and git-annex-shell on the remote system. (With either
able to be the attacker). Only those with the right ssh key can do it. And,
there are probably lots of ways to construct git repositories that make git
use a lot of memory in various ways, which would have similar impact as
this attack.

The fix in P2P/IO.hs would have been higher impact, if it had made it to a
released version, since it would have allowed DOSing the tor hidden
service without needing to authenticate.

(The LockContent and NotifyChanges instances may not be really
exploitable; since the line is read and ignored, it probably gets read
lazily and does not end up staying buffered in memory.)
---
 CHANGELOG                     |  1 +
 Command/LockContent.hs        |  3 ++-
 Command/NotifyChanges.hs      |  3 ++-
 Command/TransferInfo.hs       |  3 ++-
 P2P/IO.hs                     |  5 ++--
 Remote/Git.hs                 |  5 ++--
 RemoteDaemon/Transport/Ssh.hs |  4 ++--
 Utility/SimpleProtocol.hs     | 44 +++++++++++++++++++++++++++--------
 8 files changed, 49 insertions(+), 19 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index 99427a0d10..b741f29b95 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -23,6 +23,7 @@ git-annex (6.20161119) UNRELEASED; urgency=medium
   * rekey: Added --batch mode.
   * add: Stage modified non-large files when running in indirect mode. 
     (This was already done in v6 mode and direct mode.)
+  * git-annex-shell, remotedaemon, git remote: Fix some memory DOS attacks.
 
  -- Joey Hess <id@joeyh.name>  Mon, 21 Nov 2016 11:27:50 -0400
 
diff --git a/Command/LockContent.hs b/Command/LockContent.hs
index 35342c529b..202ba20d1c 100644
--- a/Command/LockContent.hs
+++ b/Command/LockContent.hs
@@ -10,6 +10,7 @@ module Command.LockContent where
 import Command
 import Annex.Content
 import Remote.Helper.Ssh (contentLockedMarker)
+import Utility.SimpleProtocol
 
 cmd :: Command
 cmd = noCommit $ 
@@ -37,7 +38,7 @@ start [ks] = do
 		( liftIO $ do
 			putStrLn contentLockedMarker
 			hFlush stdout
-			_ <- getLine
+			_ <- getProtocolLine stdin
 			return True
 		, return False
 		)
diff --git a/Command/NotifyChanges.hs b/Command/NotifyChanges.hs
index f1c149d542..bb9b10eee3 100644
--- a/Command/NotifyChanges.hs
+++ b/Command/NotifyChanges.hs
@@ -13,6 +13,7 @@ import Utility.DirWatcher.Types
 import qualified Git
 import Git.Sha
 import RemoteDaemon.Transport.Ssh.Types
+import Utility.SimpleProtocol
 
 import Control.Concurrent
 import Control.Concurrent.Async
@@ -52,7 +53,7 @@ start = do
 	
 	-- No messages need to be received from the caller,
 	-- but when it closes the connection, notice and terminate.
-	let receiver = forever $ void getLine
+	let receiver = forever $ void $ getProtocolLine stdin
 	void $ liftIO $ concurrently sender receiver
 	stop
 
diff --git a/Command/TransferInfo.hs b/Command/TransferInfo.hs
index 6870c84f01..1db6334841 100644
--- a/Command/TransferInfo.hs
+++ b/Command/TransferInfo.hs
@@ -13,6 +13,7 @@ import Types.Transfer
 import Logs.Transfer
 import qualified CmdLine.GitAnnexShell.Fields as Fields
 import Utility.Metered
+import Utility.SimpleProtocol
 
 cmd :: Command
 cmd = noCommit $ 
@@ -62,4 +63,4 @@ start (k:[]) = do
 start _ = giveup "wrong number of parameters"
 
 readUpdate :: IO (Maybe Integer)
-readUpdate = readish <$> getLine
+readUpdate = maybe Nothing readish <$> getProtocolLine stdin
diff --git a/P2P/IO.hs b/P2P/IO.hs
index a2af5946c0..2693558c18 100644
--- a/P2P/IO.hs
+++ b/P2P/IO.hs
@@ -102,10 +102,11 @@ runNet conn runner f = case f of
 			Left e -> return (Left (show e))
 			Right () -> runner next
 	ReceiveMessage next -> do
-		v <- liftIO $ tryNonAsync $ hGetLine (connIhdl conn)
+		v <- liftIO $ tryNonAsync $ getProtocolLine (connIhdl conn)
 		case v of
 			Left e -> return (Left (show e))
-			Right l -> case parseMessage l of
+			Right Nothing -> return (Left "protocol error")
+			Right (Just l) -> case parseMessage l of
 				Just m -> runner (next m)
 				Nothing -> runner $ do
 					let e = ERROR $ "protocol parse error: " ++ show l
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 37a9f48b3e..5eb6fbc9e5 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -45,6 +45,7 @@ import Utility.CopyFile
 #endif
 import Utility.Env
 import Utility.Batch
+import Utility.SimpleProtocol
 import Remote.Helper.Git
 import Remote.Helper.Messages
 import qualified Remote.Helper.Ssh as Ssh
@@ -390,7 +391,7 @@ lockKey r key callback
 						, std_out = CreatePipe
 						, std_err = UseHandle nullh
 						}
-		v <- liftIO $ tryIO $ hGetLine hout
+		v <- liftIO $ tryIO $ getProtocolLine hout
 		let signaldone = void $ tryNonAsync $ liftIO $ mapM_ tryNonAsync
 			[ hPutStrLn hout ""
 			, hFlush hout
@@ -408,7 +409,7 @@ lockKey r key callback
 					void $ waitForProcess p
 				failedlock
 			Right l 
-				| l == Ssh.contentLockedMarker -> bracket_
+				| l == Just Ssh.contentLockedMarker -> bracket_
 					noop
 					signaldone 
 					(withVerifiedCopy LockedCopy r checkexited callback)
diff --git a/RemoteDaemon/Transport/Ssh.hs b/RemoteDaemon/Transport/Ssh.hs
index 73c88054c6..2051650627 100644
--- a/RemoteDaemon/Transport/Ssh.hs
+++ b/RemoteDaemon/Transport/Ssh.hs
@@ -68,8 +68,8 @@ transportUsingCmd' cmd params (RemoteRepo r _) url transporthandle ichan ochan =
 		send (DONESYNCING url ok)
 		
 	handlestdout fromh = do
-		l <- hGetLine fromh
-		case parseMessage l of
+		ml <- getProtocolLine fromh
+		case parseMessage =<< ml of
 			Just SshRemote.READY -> do
 				send (CONNECTED url)
 				handlestdout fromh
diff --git a/Utility/SimpleProtocol.hs b/Utility/SimpleProtocol.hs
index 473129218a..7ab3c8c776 100644
--- a/Utility/SimpleProtocol.hs
+++ b/Utility/SimpleProtocol.hs
@@ -1,6 +1,6 @@
 {- Simple line-based protocols.
  -
- - Copyright 2013-2014 Joey Hess <id@joeyh.name>
+ - Copyright 2013-2016 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}
@@ -20,6 +20,7 @@ module Utility.SimpleProtocol (
 	parse2,
 	parse3,
 	dupIoHandles,
+	getProtocolLine,
 ) where
 
 import Data.Char
@@ -48,6 +49,16 @@ class Serializable a where
 	serialize :: a -> String
 	deserialize :: String -> Maybe a
 
+instance Serializable [Char] where
+	serialize = id
+	deserialize = Just
+
+instance Serializable ExitCode where
+	serialize ExitSuccess = "0"
+	serialize (ExitFailure n) = show n
+	deserialize "0" = Just ExitSuccess
+	deserialize s = ExitFailure <$> readish s
+
 {- Parsing the parameters of messages. Using the right parseN ensures
  - that the string is split into exactly the requested number of words,
  - which allows the last parameter of a message to contain arbitrary
@@ -93,12 +104,25 @@ dupIoHandles = do
 	stderr `hDuplicateTo` stdout
 	return (readh, writeh)
 
-instance Serializable [Char] where
-	serialize = id
-	deserialize = Just
-
-instance Serializable ExitCode where
-	serialize ExitSuccess = "0"
-	serialize (ExitFailure n) = show n
-	deserialize "0" = Just ExitSuccess
-	deserialize s = ExitFailure <$> readish s
+{- Reads a line, but to avoid super-long lines eating memory, returns
+ - Nothing if 32 kb have been read without seeing a '\n'
+ -
+ - If there is a '\r' before the '\n', it is removed, to support
+ - systems using "\r\n" at ends of lines 
+ -
+ - This implementation is not super efficient, but as long as the Handle
+ - supports buffering, it avoids reading a character at a time at the
+ - syscall level.
+ -}
+getProtocolLine :: Handle -> IO (Maybe String)
+getProtocolLine h = go (32768 :: Int) []
+  where
+	go 0 _ = return Nothing
+	go n l = do
+		c <- hGetChar h
+		if c == '\n'
+			then return $ Just $ reverse $ 
+				case l of
+					('\r':rest) -> rest
+					_ -> l
+			else go (n-1) (c:l)

From 596e1685a6aaaff0fdb32d6f1f991f4749bfbe1b Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 9 Dec 2016 13:38:50 -0400
Subject: [PATCH 242/367] update

---
 doc/todo/tor.mdwn | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index 11d441daba..0980fbaf5f 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -4,16 +4,14 @@ Mostly working!
 
 Current todo list:
 
-* Current use of hGetLine to read protocol messages allows memory DOS by
-  sending a very long line. May also have line ending problems across OS's. 
-  Switch to instead reading a packed data structure that starts with its
-  length, and refuse to read messages > 32k.
 * When a transfer can't be done because another transfer of the same 
   object is already in progress, the message about this is output by the
   remotedaemon --debug, but not forwarded to the peer, which shows
   "Connection reset by peer"
 * Think about locking some more. What happens if the connection to the peer
   is dropped while we think we're locking content there from being dropped?
+* Make remotedaemon connect to tor peers, notice when their repos have
+  changed, and pull, like it does for ssh peers.
 
 Eventually:
 

From e152c322f8d7a2430636f26aa55272773ea1c9da Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 9 Dec 2016 14:52:38 -0400
Subject: [PATCH 243/367] refactor ref change watching

Added to change notification to P2P protocol.

Switched to a TBChan so that a single long-running thread can be
started, and serve perhaps intermittent requests for change
notifications, without buffering all changes in memory.

The P2P runner currently starts up a new thread each times it waits
for a change, but that should allow later reusing a thread. Although
each connection from a peer will still need a new watcher thread to run.

The dependency on stm-chans is more or less free; some stuff in yesod
uses it, so it was already indirectly pulled in when building with the
webapp.

This commit was sponsored by Francois Marier on Patreon.
---
 Annex/ChangedRefs.hs                | 105 ++++++++++++++++++++++++++++
 Command/NotifyChanges.hs            |  48 ++-----------
 P2P/Annex.hs                        |   9 +++
 P2P/Protocol.hs                     |  13 ++++
 RemoteDaemon/Transport/Ssh.hs       |   3 +-
 RemoteDaemon/Transport/Ssh/Types.hs |   4 +-
 RemoteDaemon/Types.hs               |  10 +--
 debian/control                      |   1 +
 git-annex.cabal                     |   2 +
 9 files changed, 142 insertions(+), 53 deletions(-)
 create mode 100644 Annex/ChangedRefs.hs

diff --git a/Annex/ChangedRefs.hs b/Annex/ChangedRefs.hs
new file mode 100644
index 0000000000..0dc82d3b3c
--- /dev/null
+++ b/Annex/ChangedRefs.hs
@@ -0,0 +1,105 @@
+{- Waiting for changed git refs
+ -
+ - Copyright 2014-216 Joey Hess <id@joeyh.name>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Annex.ChangedRefs
+	( ChangedRefs(..)
+	, ChangedRefsHandle
+	, waitChangedRefs
+	, drainChangedRefs
+	, stopWatchingChangedRefs
+	, watchChangedRefs
+	) where
+
+import Annex.Common
+import Utility.DirWatcher
+import Utility.DirWatcher.Types
+import qualified Git
+import Git.Sha
+import qualified Utility.SimpleProtocol as Proto
+
+import Control.Concurrent
+import Control.Concurrent.STM
+import Control.Concurrent.STM.TBMChan
+
+newtype ChangedRefs = ChangedRefs [Git.Ref]
+	deriving (Show)
+
+instance Proto.Serializable ChangedRefs where
+	serialize (ChangedRefs l) = unwords $ map Git.fromRef l
+	deserialize = Just . ChangedRefs . map Git.Ref . words
+
+data ChangedRefsHandle = ChangedRefsHandle DirWatcherHandle (TBMChan Git.Sha)
+
+-- | Wait for one or more git refs to change.
+--
+-- When possible, coalesce ref writes that occur closely together
+-- in time. Delay up to 0.05 seconds to get more ref writes.
+waitChangedRefs :: ChangedRefsHandle -> IO ChangedRefs
+waitChangedRefs (ChangedRefsHandle _ chan) = do
+	v <- atomically $ readTBMChan chan
+	case v of
+		Nothing -> return $ ChangedRefs []
+		Just r -> do
+			threadDelay 50000
+			rs <- atomically $ loop []
+			return $ ChangedRefs (r:rs)
+  where
+	loop rs = do
+		v <- tryReadTBMChan chan
+		case v of
+			Just (Just r) -> loop (r:rs)
+			_ -> return rs
+
+-- | Remove any changes that might be buffered in the channel,
+-- without waiting for any new changes.
+drainChangedRefs :: ChangedRefsHandle -> IO ()
+drainChangedRefs (ChangedRefsHandle _ chan) = atomically go
+  where
+	go = do
+		v <- tryReadTBMChan chan
+		case v of
+			Just (Just _) -> go
+			_ -> return ()
+
+stopWatchingChangedRefs :: ChangedRefsHandle -> IO ()
+stopWatchingChangedRefs h@(ChangedRefsHandle wh chan) = do
+	stopWatchDir wh
+	atomically $ closeTBMChan chan
+	drainChangedRefs h
+
+watchChangedRefs :: Annex ChangedRefsHandle
+watchChangedRefs = do
+	-- This channel is used to accumulate notifications,
+	-- because the DirWatcher might have multiple threads that find
+	-- changes at the same time. It is bounded to allow a watcher
+	-- to be started once and reused, without too many changes being
+	-- buffered in memory.
+	chan <- liftIO $ newTBMChanIO 100
+	
+	g <- gitRepo
+	let refdir = Git.localGitDir g </> "refs"
+	liftIO $ createDirectoryIfMissing True refdir
+
+	let notifyhook = Just $ notifyHook chan
+	let hooks = mkWatchHooks
+		{ addHook = notifyhook
+		, modifyHook = notifyhook
+		}
+
+	h <- liftIO $ watchDir refdir (const False) True hooks id
+	return $ ChangedRefsHandle h chan
+
+notifyHook :: TBMChan Git.Sha -> FilePath -> Maybe FileStatus -> IO ()
+notifyHook chan reffile _
+	| ".lock" `isSuffixOf` reffile = noop
+	| otherwise = void $ do
+		sha <- catchDefaultIO Nothing $
+			extractSha <$> readFile reffile
+		-- When the channel is full, there is probably no reader
+		-- running, or ref changes have been occuring very fast,
+		-- so it's ok to not write the change to it.
+		maybe noop (void . atomically . tryWriteTBMChan chan) sha
diff --git a/Command/NotifyChanges.hs b/Command/NotifyChanges.hs
index bb9b10eee3..83d7bca3f8 100644
--- a/Command/NotifyChanges.hs
+++ b/Command/NotifyChanges.hs
@@ -8,6 +8,7 @@
 module Command.NotifyChanges where
 
 import Command
+import Annex.ChangedRefs
 import Utility.DirWatcher
 import Utility.DirWatcher.Types
 import qualified Git
@@ -30,55 +31,18 @@ seek = withNothing start
 
 start :: CommandStart
 start = do
-	-- This channel is used to accumulate notifcations,
-	-- because the DirWatcher might have multiple threads that find
-	-- changes at the same time.
-	chan <- liftIO newTChanIO
-	
-	g <- gitRepo
-	let refdir = Git.localGitDir g </> "refs"
-	liftIO $ createDirectoryIfMissing True refdir
+	h <- watchChangedRefs
 
-	let notifyhook = Just $ notifyHook chan
-	let hooks = mkWatchHooks
-		{ addHook = notifyhook
-		, modifyHook = notifyhook
-		}
-
-	void $ liftIO $ watchDir refdir (const False) True hooks id
-
-	let sender = do
-		send READY
-		forever $ send . CHANGED =<< drain chan
-	
 	-- No messages need to be received from the caller,
 	-- but when it closes the connection, notice and terminate.
 	let receiver = forever $ void $ getProtocolLine stdin
+	let sender = forever $ send . CHANGED =<< waitChangedRefs h
+
+	liftIO $ send READY
 	void $ liftIO $ concurrently sender receiver
+	liftIO $ stopWatchingChangedRefs h
 	stop
 
-notifyHook :: TChan Git.Sha -> FilePath -> Maybe FileStatus -> IO ()
-notifyHook chan reffile _
-	| ".lock" `isSuffixOf` reffile = noop
-	| otherwise = void $ do
-		sha <- catchDefaultIO Nothing $
-			extractSha <$> readFile reffile
-		maybe noop (atomically . writeTChan chan) sha
-
--- When possible, coalesce ref writes that occur closely together
--- in time. Delay up to 0.05 seconds to get more ref writes.
-drain :: TChan Git.Sha -> IO [Git.Sha]
-drain chan = do
-	r <- atomically $ readTChan chan
-	threadDelay 50000
-	rs <- atomically $ drain' chan
-	return (r:rs)
-
-drain' :: TChan Git.Sha -> STM [Git.Sha]
-drain' chan = loop []
-  where
-	loop rs = maybe (return rs) (\r -> loop (r:rs)) =<< tryReadTChan chan
-
 send :: Notification -> IO ()
 send n = do
 	putStrLn $ unwords $ formatMessage n
diff --git a/P2P/Annex.hs b/P2P/Annex.hs
index d24e65b0fc..e9b59652c0 100644
--- a/P2P/Annex.hs
+++ b/P2P/Annex.hs
@@ -16,6 +16,7 @@ module P2P.Annex
 import Annex.Common
 import Annex.Content
 import Annex.Transfer
+import Annex.ChangedRefs
 import P2P.Protocol
 import P2P.IO
 import Logs.Location
@@ -114,6 +115,14 @@ runLocal runmode runner a = case a of
 				protoaction False
 				next
 			Right _ -> runner next
+	WaitRefChange next -> do
+		v <- tryNonAsync $ bracket
+			watchChangedRefs
+			(liftIO . stopWatchingChangedRefs)
+			(liftIO . waitChangedRefs)
+		case v of
+			Left e -> return (Left (show e))
+			Right changedrefs -> runner (next changedrefs)
   where
 	transfer mk k af ta = case runmode of
 		-- Update transfer logs when serving.
diff --git a/P2P/Protocol.hs b/P2P/Protocol.hs
index 03c7c70cf7..d8be3ff426 100644
--- a/P2P/Protocol.hs
+++ b/P2P/Protocol.hs
@@ -19,6 +19,7 @@ import Utility.Applicative
 import Utility.PartialPrelude
 import Utility.Metered
 import Git.FilePath
+import Annex.ChangedRefs (ChangedRefs)
 
 import Control.Monad
 import Control.Monad.Free
@@ -50,6 +51,8 @@ data Message
 	| AUTH_FAILURE
 	| CONNECT Service
 	| CONNECTDONE ExitCode
+	| NOTIFYCHANGE
+	| CHANGED ChangedRefs
 	| CHECKPRESENT Key
 	| LOCKCONTENT Key
 	| UNLOCKCONTENT
@@ -70,6 +73,8 @@ instance Proto.Sendable Message where
 	formatMessage AUTH_FAILURE = ["AUTH-FAILURE"]
 	formatMessage (CONNECT service) = ["CONNECT", Proto.serialize service]
 	formatMessage (CONNECTDONE exitcode) = ["CONNECTDONE", Proto.serialize exitcode]
+	formatMessage NOTIFYCHANGE = ["NOTIFYCHANGE"]
+	formatMessage (CHANGED refs) = ["CHANGED", Proto.serialize refs]
 	formatMessage (CHECKPRESENT key) = ["CHECKPRESENT", Proto.serialize key]
 	formatMessage (LOCKCONTENT key) = ["LOCKCONTENT", Proto.serialize key]
 	formatMessage UNLOCKCONTENT = ["UNLOCKCONTENT"]
@@ -89,6 +94,8 @@ instance Proto.Receivable Message where
 	parseCommand "AUTH-FAILURE" = Proto.parse0 AUTH_FAILURE
 	parseCommand "CONNECT" = Proto.parse1 CONNECT
 	parseCommand "CONNECTDONE" = Proto.parse1 CONNECTDONE
+	parseCommand "NOTIFYCHANGE" = Proto.parse0 NOTIFYCHANGE
+	parseCommand "CHANGED" = Proto.parse1 CHANGED
 	parseCommand "CHECKPRESENT" = Proto.parse1 CHECKPRESENT
 	parseCommand "LOCKCONTENT" = Proto.parse1 LOCKCONTENT
 	parseCommand "UNLOCKCONTENT" = Proto.parse0 UNLOCKCONTENT
@@ -227,6 +234,8 @@ data LocalF c
 	-- from being deleted, while running the provided protocol
 	-- action. If unable to lock the content, runs the protocol action
 	-- with False.
+	| WaitRefChange (ChangedRefs -> c)
+	-- ^ Waits for one or more git refs to change and returns them.
 	deriving (Functor)
 
 type Local = Free LocalF
@@ -379,6 +388,10 @@ serveAuthed myuuid = void $ serverLoop handler
 	handler (CONNECT service) = do
 		net $ relayService service
 		return ServerContinue
+	handler NOTIFYCHANGE = do
+		refs <- local waitRefChange
+		net $ sendMessage (CHANGED refs)
+		return ServerContinue
 	handler _ = return ServerUnexpected
 
 sendContent :: Key -> AssociatedFile -> Offset -> MeterUpdate -> Proto Bool
diff --git a/RemoteDaemon/Transport/Ssh.hs b/RemoteDaemon/Transport/Ssh.hs
index 2051650627..59502f8d3c 100644
--- a/RemoteDaemon/Transport/Ssh.hs
+++ b/RemoteDaemon/Transport/Ssh.hs
@@ -17,6 +17,7 @@ import Utility.SimpleProtocol
 import qualified Git
 import Git.Command
 import Utility.ThreadScheduler
+import Annex.ChangedRefs
 
 import Control.Concurrent.STM
 import Control.Concurrent.Async
@@ -73,7 +74,7 @@ transportUsingCmd' cmd params (RemoteRepo r _) url transporthandle ichan ochan =
 			Just SshRemote.READY -> do
 				send (CONNECTED url)
 				handlestdout fromh
-			Just (SshRemote.CHANGED shas) -> do
+			Just (SshRemote.CHANGED (ChangedRefs shas)) -> do
 				whenM (checkNewShas transporthandle shas) $
 					fetch
 				handlestdout fromh
diff --git a/RemoteDaemon/Transport/Ssh/Types.hs b/RemoteDaemon/Transport/Ssh/Types.hs
index fa6a55d3dd..606e1a563b 100644
--- a/RemoteDaemon/Transport/Ssh/Types.hs
+++ b/RemoteDaemon/Transport/Ssh/Types.hs
@@ -16,11 +16,11 @@ module RemoteDaemon.Transport.Ssh.Types (
 ) where
 
 import qualified Utility.SimpleProtocol as Proto
-import RemoteDaemon.Types (RefList)
+import Annex.ChangedRefs (ChangedRefs)
 
 data Notification
 	= READY
-	| CHANGED RefList
+	| CHANGED ChangedRefs
 
 instance Proto.Sendable Notification where
 	formatMessage READY = ["READY"]
diff --git a/RemoteDaemon/Types.hs b/RemoteDaemon/Types.hs
index ba88aa685b..c0d74e038a 100644
--- a/RemoteDaemon/Types.hs
+++ b/RemoteDaemon/Types.hs
@@ -5,7 +5,6 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-{-# LANGUAGE TypeSynonymInstances, FlexibleInstances #-}
 {-# OPTIONS_GHC -fno-warn-orphans #-}
 
 module RemoteDaemon.Types where
@@ -15,6 +14,7 @@ import qualified Annex
 import qualified Git.Types as Git
 import qualified Utility.SimpleProtocol as Proto
 import Types.GitConfig
+import Annex.ChangedRefs (ChangedRefs)
 
 import Network.URI
 import Control.Concurrent
@@ -52,13 +52,11 @@ data Consumed
 	= PAUSE
 	| LOSTNET
 	| RESUME
-	| CHANGED RefList
+	| CHANGED ChangedRefs
 	| RELOAD
 	| STOP
 	deriving (Show)
 
-type RefList = [Git.Ref]
-
 instance Proto.Sendable Emitted where
 	formatMessage (CONNECTED remote) =
 		["CONNECTED", Proto.serialize remote]
@@ -100,10 +98,6 @@ instance Proto.Serializable RemoteURI where
 	serialize (RemoteURI u) = show u
 	deserialize = RemoteURI <$$> parseURI
 
-instance Proto.Serializable RefList where
-	serialize = unwords . map Git.fromRef
-	deserialize = Just . map Git.Ref . words
-
 instance Proto.Serializable Bool where
 	serialize False = "0"
 	serialize True = "1"
diff --git a/debian/control b/debian/control
index 1d2313954a..f5a9de8408 100644
--- a/debian/control
+++ b/debian/control
@@ -50,6 +50,7 @@ Build-Depends:
 	libghc-esqueleto-dev,
 	libghc-securemem-dev,
 	libghc-byteable-dev,
+	libghc-stm-chans-dev,
 	libghc-dns-dev,
 	libghc-case-insensitive-dev,
 	libghc-http-types-dev,
diff --git a/git-annex.cabal b/git-annex.cabal
index 465769ea0e..ec54a146d9 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -371,6 +371,7 @@ Executable git-annex
    regex-tdfa,
    socks,
    byteable,
+   stm-chans,
    securemem
   CC-Options: -Wall
   GHC-Options: -Wall -fno-warn-tabs
@@ -513,6 +514,7 @@ Executable git-annex
     Annex.Branch.Transitions
     Annex.BranchState
     Annex.CatFile
+    Annex.ChangedRefs
     Annex.CheckAttr
     Annex.CheckIgnore
     Annex.Common

From f7687e087619abc31a06e337e01d00eec84c1798 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 9 Dec 2016 15:08:54 -0400
Subject: [PATCH 244/367] only start ref change watcher thread once per P2P
 connection

This is more efficient. Note that the peer will get CHANGED messages for
all refs changed since the connection opened, even if those changes
happened before it sent NOTIFYCHANGE.
---
 P2P/Annex.hs                  | 21 +++++++++------------
 RemoteDaemon/Transport/Tor.hs | 31 ++++++++++++++++++++-----------
 2 files changed, 29 insertions(+), 23 deletions(-)

diff --git a/P2P/Annex.hs b/P2P/Annex.hs
index e9b59652c0..351fb38bb7 100644
--- a/P2P/Annex.hs
+++ b/P2P/Annex.hs
@@ -25,10 +25,8 @@ import Utility.Metered
 
 import Control.Monad.Free
 
--- When we're serving a peer, we know their uuid, and can use it to update
--- transfer logs.
 data RunMode
-	= Serving UUID
+	= Serving UUID ChangedRefsHandle
 	| Client
 
 -- Full interpreter for Proto, that can receive and send objects.
@@ -115,18 +113,17 @@ runLocal runmode runner a = case a of
 				protoaction False
 				next
 			Right _ -> runner next
-	WaitRefChange next -> do
-		v <- tryNonAsync $ bracket
-			watchChangedRefs
-			(liftIO . stopWatchingChangedRefs)
-			(liftIO . waitChangedRefs)
-		case v of
-			Left e -> return (Left (show e))
-			Right changedrefs -> runner (next changedrefs)
+	WaitRefChange next -> case runmode of
+		Serving _ h -> do
+			v <- tryNonAsync $ liftIO $ waitChangedRefs h
+			case v of
+				Left e -> return (Left (show e))
+				Right changedrefs -> runner (next changedrefs)
+		_ -> return $ Left "change notification not implemented for client"
   where
 	transfer mk k af ta = case runmode of
 		-- Update transfer logs when serving.
-		Serving theiruuid -> 
+		Serving theiruuid _ -> 
 			mk theiruuid k af noRetry ta noNotification
 		-- Transfer logs are updated higher in the stack when
 		-- a client.
diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs
index 0dd1d1ba27..5ea06ac2c3 100644
--- a/RemoteDaemon/Transport/Tor.hs
+++ b/RemoteDaemon/Transport/Tor.hs
@@ -10,6 +10,7 @@ module RemoteDaemon.Transport.Tor (server) where
 import Common
 import qualified Annex
 import Annex.Concurrent
+import Annex.ChangedRefs
 import RemoteDaemon.Types
 import RemoteDaemon.Common
 import Utility.Tor
@@ -71,12 +72,18 @@ maxConnections :: Int
 maxConnections = 10
 
 serveClient :: TransportHandle -> UUID -> Repo -> TBQueue Handle -> IO ()
-serveClient th u r q = bracket setup cleanup go
+serveClient th u r q = bracket setup cleanup start
   where
-	setup = atomically $ readTBQueue q
-	cleanup = hClose
-	go h = do
+	setup = do
+		h <- atomically $ readTBQueue q
 		debugM "remotedaemon" "serving a Tor connection"
+		return h
+	
+	cleanup h = do
+		debugM "remotedaemon" "done with Tor connection"
+		hClose h
+
+	start h = do
 		-- Avoid doing any work in the liftAnnex, since only one
 		-- can run at a time.
 		st <- liftAnnex th dupState
@@ -92,16 +99,18 @@ serveClient th u r q = bracket setup cleanup go
 				}
 			v <- liftIO $ runNetProto conn $ serveAuth u
 			case v of
-				Right (Just theiruuid) -> void $ do
-					v' <- runFullProto (Serving theiruuid) conn $
-						serveAuthed u
-					case v' of
-						Right () -> return ()
-						Left e -> liftIO $ debugM "remotedaemon" ("Tor connection error: " ++ e)
+				Right (Just theiruuid) -> authed conn theiruuid
 				Right Nothing -> liftIO $
 					debugM "remotedaemon" "Tor connection failed to authenticate"
 				Left e -> liftIO $
 					debugM "remotedaemon" ("Tor connection error before authentication: " ++ e)
 		-- Merge the duplicated state back in.
 		liftAnnex th $ mergeState st'
-		debugM "remotedaemon" "done with Tor connection"
+	
+	authed conn theiruuid = 
+		bracket watchChangedRefs (liftIO . stopWatchingChangedRefs) $ \crh -> do
+			v' <- runFullProto (Serving theiruuid crh) conn $
+				serveAuthed u
+			case v' of
+				Right () -> return ()
+				Left e -> liftIO $ debugM "remotedaemon" ("Tor connection error: " ++ e)

From 2c907fff51778c821ef4c5ce247d63886a1337ee Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 9 Dec 2016 16:02:43 -0400
Subject: [PATCH 245/367] remotedaemon: git change detection over tor hidden
 service

---
 P2P/Address.hs                  |  7 +++-
 P2P/Protocol.hs                 | 10 +++++
 RemoteDaemon/Common.hs          | 24 ++++++++++-
 RemoteDaemon/Transport.hs       |  2 +
 RemoteDaemon/Transport/Ssh.hs   | 29 +++----------
 RemoteDaemon/Transport/Tor.hs   | 73 ++++++++++++++++++++++++++++-----
 doc/git-annex-remotedaemon.mdwn | 14 +++----
 doc/todo/tor.mdwn               |  3 +-
 8 files changed, 116 insertions(+), 46 deletions(-)

diff --git a/P2P/Address.hs b/P2P/Address.hs
index 09ffc79735..1b1f66059e 100644
--- a/P2P/Address.hs
+++ b/P2P/Address.hs
@@ -37,15 +37,18 @@ class FormatP2PAddress a where
 
 instance FormatP2PAddress P2PAddress where
 	formatP2PAddress (TorAnnex (OnionAddress onionaddr) onionport) =
-		"tor-annex::" ++ onionaddr ++ ":" ++ show onionport
+		torAnnexScheme ++ ":" ++ onionaddr ++ ":" ++ show onionport
 	unformatP2PAddress s
-		| "tor-annex::" `isPrefixOf` s = do
+		| (torAnnexScheme ++ ":") `isPrefixOf` s = do
 			let s' = dropWhile (== ':') $ dropWhile (/= ':') s
 			let (onionaddr, ps) = separate (== ':') s'
 			onionport <- readish ps
 			return (TorAnnex (OnionAddress onionaddr) onionport)
 		| otherwise = Nothing
 
+torAnnexScheme :: String
+torAnnexScheme = "tor-annex:"
+
 instance FormatP2PAddress P2PAddressAuth where
 	formatP2PAddress (P2PAddressAuth addr authtoken) =
 		formatP2PAddress addr ++ ":" ++ T.unpack (fromAuthToken authtoken)
diff --git a/P2P/Protocol.hs b/P2P/Protocol.hs
index d8be3ff426..c3c362f37d 100644
--- a/P2P/Protocol.hs
+++ b/P2P/Protocol.hs
@@ -441,6 +441,16 @@ sendSuccess :: Bool -> Proto ()
 sendSuccess True = net $ sendMessage SUCCESS
 sendSuccess False = net $ sendMessage FAILURE
 
+notifyChange :: Proto (Maybe ChangedRefs)
+notifyChange = do
+	net $ sendMessage NOTIFYCHANGE
+	ack <- net receiveMessage
+	case ack of
+		CHANGED rs -> return (Just rs)
+		_ -> do
+			net $ sendMessage (ERROR "expected CHANGED")
+			return Nothing
+
 connect :: Service -> Handle -> Handle -> Proto ExitCode
 connect service hin hout = do
 	net $ sendMessage (CONNECT service)
diff --git a/RemoteDaemon/Common.hs b/RemoteDaemon/Common.hs
index 982a84b439..711771f974 100644
--- a/RemoteDaemon/Common.hs
+++ b/RemoteDaemon/Common.hs
@@ -1,6 +1,6 @@
 {- git-remote-daemon utilities
  -
- - Copyright 2014 Joey Hess <id@joeyh.name>
+ - Copyright 2014-2016 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -9,6 +9,8 @@ module RemoteDaemon.Common
 	( liftAnnex
 	, inLocalRepo
 	, checkNewShas
+	, ConnectionStatus(..)
+	, robustConnection
 	) where
 
 import qualified Annex
@@ -16,6 +18,7 @@ import Annex.Common
 import RemoteDaemon.Types
 import qualified Git
 import Annex.CatFile
+import Utility.ThreadScheduler
 
 import Control.Concurrent
 
@@ -40,3 +43,22 @@ checkNewShas transporthandle = check
 	check [] = return True
 	check (r:rs) = maybe (check rs) (const $ return False)
 		=<< liftAnnex transporthandle (catObjectDetails r)
+
+data ConnectionStatus = ConnectionStopping | ConnectionClosed
+
+{- Make connection robust, retrying on error, with exponential backoff. -}
+robustConnection :: Int -> IO ConnectionStatus -> IO ()
+robustConnection backoff a = 
+	caught =<< a `catchNonAsync` (const $ return ConnectionClosed)
+  where
+	caught ConnectionStopping = return ()
+	caught ConnectionClosed = do
+		threadDelaySeconds (Seconds backoff)
+		robustConnection increasedbackoff a
+
+	increasedbackoff
+		| b2 > maxbackoff = maxbackoff
+		| otherwise = b2
+	  where
+		b2 = backoff * 2
+		maxbackoff = 3600 -- one hour
diff --git a/RemoteDaemon/Transport.hs b/RemoteDaemon/Transport.hs
index 6605012de3..053973424f 100644
--- a/RemoteDaemon/Transport.hs
+++ b/RemoteDaemon/Transport.hs
@@ -12,6 +12,7 @@ import qualified RemoteDaemon.Transport.Ssh
 import qualified RemoteDaemon.Transport.GCrypt
 import qualified RemoteDaemon.Transport.Tor
 import qualified Git.GCrypt
+import P2P.Address (torAnnexScheme)
 
 import qualified Data.Map as M
 
@@ -22,6 +23,7 @@ remoteTransports :: M.Map TransportScheme Transport
 remoteTransports = M.fromList
 	[ ("ssh:", RemoteDaemon.Transport.Ssh.transport)
 	, (Git.GCrypt.urlScheme, RemoteDaemon.Transport.GCrypt.transport)
+	, (torAnnexScheme, RemoteDaemon.Transport.Tor.transport)
 	]
 
 remoteServers :: [TransportHandle -> IO ()]
diff --git a/RemoteDaemon/Transport/Ssh.hs b/RemoteDaemon/Transport/Ssh.hs
index 59502f8d3c..6f8e8323e8 100644
--- a/RemoteDaemon/Transport/Ssh.hs
+++ b/RemoteDaemon/Transport/Ssh.hs
@@ -16,7 +16,6 @@ import qualified RemoteDaemon.Transport.Ssh.Types as SshRemote
 import Utility.SimpleProtocol
 import qualified Git
 import Git.Command
-import Utility.ThreadScheduler
 import Annex.ChangedRefs
 
 import Control.Concurrent.STM
@@ -38,7 +37,7 @@ transportUsingCmd cmd params rr@(RemoteRepo r gc) url h@(TransportHandle (LocalR
 
 transportUsingCmd' :: FilePath -> [CommandParam] -> Transport
 transportUsingCmd' cmd params (RemoteRepo r _) url transporthandle ichan ochan =
-	robustly 1 $ do
+	robustConnection 1 $ do
 		(Just toh, Just fromh, Just errh, pid) <-
 			createProcess (proc cmd (toCommand params))
 			{ std_in = CreatePipe
@@ -79,13 +78,13 @@ transportUsingCmd' cmd params (RemoteRepo r _) url transporthandle ichan ochan =
 					fetch
 				handlestdout fromh
 			-- avoid reconnect on protocol error
-			Nothing -> return Stopping
+			Nothing -> return ConnectionStopping
 	
 	handlecontrol = do
 		msg <- atomically $ readTChan ichan
 		case msg of
-			STOP -> return Stopping
-			LOSTNET -> return Stopping
+			STOP -> return ConnectionStopping
+			LOSTNET -> return ConnectionStopping
 			_ -> handlecontrol
 
 	-- Old versions of git-annex-shell that do not support
@@ -103,23 +102,5 @@ transportUsingCmd' cmd params (RemoteRepo r _) url transporthandle ichan ochan =
 					, "needs its git-annex upgraded"
 					, "to 5.20140405 or newer"
 					]
-				return Stopping
+				return ConnectionStopping
 			else handlestderr errh
-
-data Status = Stopping | ConnectionClosed
-
-{- Make connection robustly, with exponential backoff on failure. -}
-robustly :: Int -> IO Status -> IO ()
-robustly backoff a = caught =<< catchDefaultIO ConnectionClosed a
-  where
-	caught Stopping = return ()
-	caught ConnectionClosed = do
-		threadDelaySeconds (Seconds backoff)
-		robustly increasedbackoff a
-
-	increasedbackoff
-		| b2 > maxbackoff = maxbackoff
-		| otherwise = b2
-	  where
-		b2 = backoff * 2
-		maxbackoff = 3600 -- one hour
diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs
index 5ea06ac2c3..20320caddd 100644
--- a/RemoteDaemon/Transport/Tor.hs
+++ b/RemoteDaemon/Transport/Tor.hs
@@ -1,11 +1,11 @@
-{- git-remote-daemon, tor hidden service transport
+{- git-remote-daemon, tor hidden service server and transport
  -
  - Copyright 2016 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module RemoteDaemon.Transport.Tor (server) where
+module RemoteDaemon.Transport.Tor (server, transport) where
 
 import Common
 import qualified Annex
@@ -16,20 +16,23 @@ import RemoteDaemon.Common
 import Utility.Tor
 import Utility.FileMode
 import Utility.AuthToken
-import P2P.Protocol
+import P2P.Protocol as P2P
 import P2P.IO
 import P2P.Annex
 import P2P.Auth
+import P2P.Address
 import Annex.UUID
 import Types.UUID
 import Messages
 import Git
+import Git.Command
 
 import System.PosixCompat.User
-import Network.Socket
 import Control.Concurrent
 import System.Log.Logger (debugM)
 import Control.Concurrent.STM
+import Control.Concurrent.Async
+import qualified Network.Socket as S
 
 -- Run tor hidden service.
 server :: TransportHandle -> IO ()
@@ -44,17 +47,17 @@ server th@(TransportHandle (LocalRepo r) _) = do
 	let ident = fromUUID u
 	let sock = hiddenServiceSocketFile uid ident
 	nukeFile sock
-	soc <- socket AF_UNIX Stream defaultProtocol
-	bind soc (SockAddrUnix sock)
+	soc <- S.socket S.AF_UNIX S.Stream S.defaultProtocol
+	S.bind soc (S.SockAddrUnix sock)
 	-- Allow everyone to read and write to the socket; tor is probably
 	-- running as a different user. Connections have to authenticate
 	-- to do anything, so it's fine that other local users can connect.
 	modifyFileMode sock $ addModes
 		[groupReadMode, groupWriteMode, otherReadMode, otherWriteMode]
-	listen soc 2
+	S.listen soc 2
 	debugM "remotedaemon" "Tor hidden service running"
 	forever $ do
-		(conn, _) <- accept soc
+		(conn, _) <- S.accept soc
 		h <- setupHandle conn
 		ok <- atomically $ ifM (isFullTBQueue q)
 			( return False
@@ -97,7 +100,7 @@ serveClient th u r q = bracket setup cleanup start
 				, connIhdl = h
 				, connOhdl = h
 				}
-			v <- liftIO $ runNetProto conn $ serveAuth u
+			v <- liftIO $ runNetProto conn $ P2P.serveAuth u
 			case v of
 				Right (Just theiruuid) -> authed conn theiruuid
 				Right Nothing -> liftIO $
@@ -110,7 +113,57 @@ serveClient th u r q = bracket setup cleanup start
 	authed conn theiruuid = 
 		bracket watchChangedRefs (liftIO . stopWatchingChangedRefs) $ \crh -> do
 			v' <- runFullProto (Serving theiruuid crh) conn $
-				serveAuthed u
+				P2P.serveAuthed u
 			case v' of
 				Right () -> return ()
 				Left e -> liftIO $ debugM "remotedaemon" ("Tor connection error: " ++ e)
+
+-- Connect to peer's tor hidden service.
+transport :: Transport
+transport (RemoteRepo r _) url@(RemoteURI uri) th ichan ochan =
+	case unformatP2PAddress (show uri) of
+		Nothing -> return ()
+		Just addr -> robustConnection 1 $ do
+			g <- liftAnnex th Annex.gitRepo
+			bracket (connectPeer g addr) closeConnection (go addr)
+  where
+	go addr conn = do
+		myuuid <- liftAnnex th getUUID
+		authtoken <- fromMaybe nullAuthToken
+			<$> liftAnnex th (loadP2PRemoteAuthToken addr)
+		res <- runNetProto conn $
+			P2P.auth myuuid authtoken
+		case res of
+			Right (Just theiruuid)
+				| getUncachedUUID r == theiruuid -> do
+					send (CONNECTED url)
+					status <- handlecontrol
+						`race` handlepeer conn
+					send (DISCONNECTED url)
+					return $ either id id status
+				| otherwise -> return ConnectionStopping
+			_ -> return ConnectionClosed
+	
+	send msg = atomically $ writeTChan ochan msg
+	
+	handlecontrol = do
+		msg <- atomically $ readTChan ichan
+		case msg of
+			STOP -> return ConnectionStopping
+			LOSTNET -> return ConnectionStopping
+			_ -> handlecontrol
+
+	handlepeer conn = do
+		v <- runNetProto conn P2P.notifyChange
+		case v of
+			Right (Just (ChangedRefs shas)) -> do
+				whenM (checkNewShas th shas) $
+					fetch
+				handlepeer conn
+			_ -> return ConnectionClosed
+	
+	fetch = do
+		send (SYNCING url)
+		ok <- inLocalRepo th $
+			runBool [Param "fetch", Param $ Git.repoDescribe r]
+		send (DONESYNCING url ok)
diff --git a/doc/git-annex-remotedaemon.mdwn b/doc/git-annex-remotedaemon.mdwn
index d4960c4ff5..b01002dc95 100644
--- a/doc/git-annex-remotedaemon.mdwn
+++ b/doc/git-annex-remotedaemon.mdwn
@@ -9,8 +9,8 @@ git annex remotedaemon
 # DESCRIPTION
 
 The remotedaemon provides persistent communication with remotes.
-This is useful to detect when remotes have received git pushes, so the
-changes can be promptly fetched and the local repository updated.
+It detects when git branches on remotes have changes, and fetches
+the changes from them.
 
 The assistant runs the remotedaemon and communicates with it on
 stdio using a simple textual protocol.
@@ -19,12 +19,12 @@ Several types of remotes are supported:
 
 For ssh remotes, the remotedaemon tries to maintain a connection to the
 remote git repository, and uses git-annex-shell notifychanges to detect
-when the remote git repository has changed, and fetch the changes from
-it. For this to work, the git remote must have [[git-annex-shell]](1)
-installed, with notifychanges support. The first version of git-annex-shell
-that supports it is 5.20140405.
+when the remote git repository has changed. For this to work, the git
+remote must have [[git-annex-shell]](1) installed, with notifychanges
+support. The first version of git-annex-shell that supports it is
+5.20140405.
 
-For tor-annex remotes, the remotedaemon runs as a tor hidden service,
+For tor-annex remotes, the remotedaemon runs a tor hidden service,
 accepting connections from other nodes and serving up the contents of the
 repository. This is only done if you first run `git annex enable-tor`.
 Use `git annex p2p` to configure access to tor-annex remotes.
diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index 0980fbaf5f..9bbc257ad4 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -10,8 +10,7 @@ Current todo list:
   "Connection reset by peer"
 * Think about locking some more. What happens if the connection to the peer
   is dropped while we think we're locking content there from being dropped?
-* Make remotedaemon connect to tor peers, notice when their repos have
-  changed, and pull, like it does for ssh peers.
+* test remotedaemon's change detection
 
 Eventually:
 

From c51d7dbf67b65a3da698f989a6fc0fe815fd2713 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 9 Dec 2016 16:03:25 -0400
Subject: [PATCH 246/367] bump maxConnections to 100

10 seemed too low because more than 10 friends could be linked to a repo
over tor, and if all were running the remotedaemon, which makes a
persistent connection for change notification, then the 11th friend
would not be able to access that repo.

100 might be too low, but it's a much larger group of people. And at
that size group, it probably makes sense to structure the network so
that 100 peers are not all trying to access one central node.
---
 RemoteDaemon/Transport/Tor.hs | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs
index 20320caddd..344d5aefbc 100644
--- a/RemoteDaemon/Transport/Tor.hs
+++ b/RemoteDaemon/Transport/Tor.hs
@@ -69,10 +69,9 @@ server th@(TransportHandle (LocalRepo r) _) = do
 			hClose h
 			warningIO "dropped Tor connection, too busy"
 
--- How many clients to serve at a time, maximum. This is to avoid DOS
--- attacks.
+-- How many clients to serve at a time, maximum. This is to avoid DOS attacks.
 maxConnections :: Int
-maxConnections = 10
+maxConnections = 100
 
 serveClient :: TransportHandle -> UUID -> Repo -> TBQueue Handle -> IO ()
 serveClient th u r q = bracket setup cleanup start

From 8ac4126bd2e1b1362d22149bc44a3f479ff3861d Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 9 Dec 2016 16:22:00 -0400
Subject: [PATCH 247/367] cleanup

---
 Command/NotifyChanges.hs | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/Command/NotifyChanges.hs b/Command/NotifyChanges.hs
index 83d7bca3f8..92c70f05eb 100644
--- a/Command/NotifyChanges.hs
+++ b/Command/NotifyChanges.hs
@@ -9,16 +9,10 @@ module Command.NotifyChanges where
 
 import Command
 import Annex.ChangedRefs
-import Utility.DirWatcher
-import Utility.DirWatcher.Types
-import qualified Git
-import Git.Sha
 import RemoteDaemon.Transport.Ssh.Types
 import Utility.SimpleProtocol
 
-import Control.Concurrent
 import Control.Concurrent.Async
-import Control.Concurrent.STM
 
 cmd :: Command
 cmd = noCommit $ 

From 9dd510bf29e084434ec908920f3e3ff1941c9f0c Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 9 Dec 2016 16:27:20 -0400
Subject: [PATCH 248/367] make tor hidden service work when directory watching
 is not available

Avoid crashing when built w/o inotify..
---
 Annex/ChangedRefs.hs          |  9 ++++++---
 Command/NotifyChanges.hs      | 23 ++++++++++++-----------
 P2P/Annex.hs                  |  6 +++---
 RemoteDaemon/Transport/Tor.hs |  2 +-
 4 files changed, 22 insertions(+), 18 deletions(-)

diff --git a/Annex/ChangedRefs.hs b/Annex/ChangedRefs.hs
index 0dc82d3b3c..1f2372c048 100644
--- a/Annex/ChangedRefs.hs
+++ b/Annex/ChangedRefs.hs
@@ -71,7 +71,7 @@ stopWatchingChangedRefs h@(ChangedRefsHandle wh chan) = do
 	atomically $ closeTBMChan chan
 	drainChangedRefs h
 
-watchChangedRefs :: Annex ChangedRefsHandle
+watchChangedRefs :: Annex (Maybe ChangedRefsHandle)
 watchChangedRefs = do
 	-- This channel is used to accumulate notifications,
 	-- because the DirWatcher might have multiple threads that find
@@ -90,8 +90,11 @@ watchChangedRefs = do
 		, modifyHook = notifyhook
 		}
 
-	h <- liftIO $ watchDir refdir (const False) True hooks id
-	return $ ChangedRefsHandle h chan
+	if canWatch
+		then do
+			h <- liftIO $ watchDir refdir (const False) True hooks id
+			return $ Just $ ChangedRefsHandle h chan
+		else return Nothing
 
 notifyHook :: TBMChan Git.Sha -> FilePath -> Maybe FileStatus -> IO ()
 notifyHook chan reffile _
diff --git a/Command/NotifyChanges.hs b/Command/NotifyChanges.hs
index 92c70f05eb..27db8ad82f 100644
--- a/Command/NotifyChanges.hs
+++ b/Command/NotifyChanges.hs
@@ -24,18 +24,19 @@ seek :: CmdParams -> CommandSeek
 seek = withNothing start
 
 start :: CommandStart
-start = do
-	h <- watchChangedRefs
+start = go =<< watchChangedRefs
+  where
+	go (Just h) = do
+		-- No messages need to be received from the caller,
+		-- but when it closes the connection, notice and terminate.
+		let receiver = forever $ void $ getProtocolLine stdin
+		let sender = forever $ send . CHANGED =<< waitChangedRefs h
 
-	-- No messages need to be received from the caller,
-	-- but when it closes the connection, notice and terminate.
-	let receiver = forever $ void $ getProtocolLine stdin
-	let sender = forever $ send . CHANGED =<< waitChangedRefs h
-
-	liftIO $ send READY
-	void $ liftIO $ concurrently sender receiver
-	liftIO $ stopWatchingChangedRefs h
-	stop
+		liftIO $ send READY
+		void $ liftIO $ concurrently sender receiver
+		liftIO $ stopWatchingChangedRefs h
+		stop
+	go Nothing = stop
 
 send :: Notification -> IO ()
 send n = do
diff --git a/P2P/Annex.hs b/P2P/Annex.hs
index 351fb38bb7..b3db7513c0 100644
--- a/P2P/Annex.hs
+++ b/P2P/Annex.hs
@@ -26,7 +26,7 @@ import Utility.Metered
 import Control.Monad.Free
 
 data RunMode
-	= Serving UUID ChangedRefsHandle
+	= Serving UUID (Maybe ChangedRefsHandle)
 	| Client
 
 -- Full interpreter for Proto, that can receive and send objects.
@@ -114,12 +114,12 @@ runLocal runmode runner a = case a of
 				next
 			Right _ -> runner next
 	WaitRefChange next -> case runmode of
-		Serving _ h -> do
+		Serving _ (Just h) -> do
 			v <- tryNonAsync $ liftIO $ waitChangedRefs h
 			case v of
 				Left e -> return (Left (show e))
 				Right changedrefs -> runner (next changedrefs)
-		_ -> return $ Left "change notification not implemented for client"
+		_ -> return $ Left "change notification not available"
   where
 	transfer mk k af ta = case runmode of
 		-- Update transfer logs when serving.
diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs
index 344d5aefbc..6149df3767 100644
--- a/RemoteDaemon/Transport/Tor.hs
+++ b/RemoteDaemon/Transport/Tor.hs
@@ -110,7 +110,7 @@ serveClient th u r q = bracket setup cleanup start
 		liftAnnex th $ mergeState st'
 	
 	authed conn theiruuid = 
-		bracket watchChangedRefs (liftIO . stopWatchingChangedRefs) $ \crh -> do
+		bracket watchChangedRefs (liftIO . maybe noop stopWatchingChangedRefs) $ \crh -> do
 			v' <- runFullProto (Serving theiruuid crh) conn $
 				P2P.serveAuthed u
 			case v' of

From 217c3b0a2153883ab74757d6273c0eb77f492fa0 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 9 Dec 2016 16:45:36 -0400
Subject: [PATCH 249/367] debug dump P2P messages

---
 P2P/IO.hs | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/P2P/IO.hs b/P2P/IO.hs
index 2693558c18..72202c2a22 100644
--- a/P2P/IO.hs
+++ b/P2P/IO.hs
@@ -40,6 +40,7 @@ import Control.Concurrent
 import Control.Concurrent.Async
 import qualified Data.ByteString as B
 import qualified Data.ByteString.Lazy as L
+import System.Log.Logger (debugM)
 
 -- Type of interpreters of the Proto free monad.
 type RunProto m = forall a. (MonadIO m, MonadMask m) => Proto a -> m (Either String a)
@@ -96,7 +97,9 @@ runNet :: (MonadIO m, MonadMask m) => P2PConnection -> RunProto m -> NetF (Proto
 runNet conn runner f = case f of
 	SendMessage m next -> do
 		v <- liftIO $ tryNonAsync $ do
-			hPutStrLn (connOhdl conn) (unwords (formatMessage m))
+			let l = unwords (formatMessage m)
+			debugM "p2p" ("P2P > " ++ l)
+			hPutStrLn (connOhdl conn) l
 			hFlush (connOhdl conn)
 		case v of
 			Left e -> return (Left (show e))
@@ -106,12 +109,14 @@ runNet conn runner f = case f of
 		case v of
 			Left e -> return (Left (show e))
 			Right Nothing -> return (Left "protocol error")
-			Right (Just l) -> case parseMessage l of
-				Just m -> runner (next m)
-				Nothing -> runner $ do
-					let e = ERROR $ "protocol parse error: " ++ show l
-					net $ sendMessage e
-					next e
+			Right (Just l) -> do
+				liftIO $ debugM "p2p" ("P2P < " ++ l)
+				case parseMessage l of
+					Just m -> runner (next m)
+					Nothing -> runner $ do
+						let e = ERROR $ "protocol parse error: " ++ show l
+						net $ sendMessage e
+						next e
 	SendBytes len b p next -> do
 		v <- liftIO $ tryNonAsync $ do
 			ok <- sendExactly len b (connOhdl conn) p

From e71755abc98e3354fc3964bdff93f44026aa47b5 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 9 Dec 2016 16:46:37 -0400
Subject: [PATCH 250/367] devblog

---
 doc/devblog/day_435-436_post_tor_merge.mdwn | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 doc/devblog/day_435-436_post_tor_merge.mdwn

diff --git a/doc/devblog/day_435-436_post_tor_merge.mdwn b/doc/devblog/day_435-436_post_tor_merge.mdwn
new file mode 100644
index 0000000000..2f05e02521
--- /dev/null
+++ b/doc/devblog/day_435-436_post_tor_merge.mdwn
@@ -0,0 +1,20 @@
+More improvements to tor support. Yesterday, debugged a reversion that
+broke push/pull over tor, and made actual useful error messages be
+displayed when there were problems. Also fixed a memory leak, although I
+fixed it by reorganizing code and could not figure out quite why it happened,
+other than that the ghc runtime was not managing to be as lazy as I would
+expect. 
+
+Today, added git ref change notification to the
+P2P protocol, and made the remotedaemon automatically fetch changes from
+tor remotes. So, it should work to use the assistant to keep
+repositories in sync over tor. I have not tried it yet, and linking over tor
+still needs to be done at the command line, so it's not really ready for
+webapp users yet.
+
+Also fixed a denial of service attack in git-annex-shell and git-annex when
+talking to a remote git-annex-shell. It was possible to feed either a large
+amount of data when they tried to read a line of data, and summon the OOM
+killer. Next release will be expedited some because of that.
+
+Today's work was sponsored by Thomas Hochstein on Patreon.

From 52ccd44812a67af2c015f0dbcc5a867bb74b6ded Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 9 Dec 2016 16:55:48 -0400
Subject: [PATCH 251/367] avoid exposing auth tokens in debug

---
 P2P/IO.hs | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/P2P/IO.hs b/P2P/IO.hs
index 72202c2a22..bb93528a65 100644
--- a/P2P/IO.hs
+++ b/P2P/IO.hs
@@ -98,7 +98,7 @@ runNet conn runner f = case f of
 	SendMessage m next -> do
 		v <- liftIO $ tryNonAsync $ do
 			let l = unwords (formatMessage m)
-			debugM "p2p" ("P2P > " ++ l)
+			debugMessage "P2P >" m
 			hPutStrLn (connOhdl conn) l
 			hFlush (connOhdl conn)
 		case v of
@@ -109,10 +109,10 @@ runNet conn runner f = case f of
 		case v of
 			Left e -> return (Left (show e))
 			Right Nothing -> return (Left "protocol error")
-			Right (Just l) -> do
-				liftIO $ debugM "p2p" ("P2P < " ++ l)
-				case parseMessage l of
-					Just m -> runner (next m)
+			Right (Just l) -> case parseMessage l of
+					Just m -> do
+						liftIO $ debugMessage "P2P <" m
+						runner (next m)
 					Nothing -> runner $ do
 						let e = ERROR $ "protocol parse error: " ++ show l
 						net $ sendMessage e
@@ -150,6 +150,14 @@ runNet conn runner f = case f of
 	-- all Proto actions.
 	runnerio = runNetProto conn
 
+debugMessage :: String -> Message -> IO ()
+debugMessage prefix m = debugM "p2p" $
+	prefix ++ " " ++ unwords (formatMessage safem)
+  where
+	safem = case m of
+		AUTH u _ -> AUTH u nullAuthToken
+		_ -> m
+
 -- Send exactly the specified number of bytes or returns False.
 --
 -- The ByteString can be larger or smaller than the specified length.

From 725b8a5e14b5c2161f982d6e1b7d06f6952d56c1 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 9 Dec 2016 17:02:21 -0400
Subject: [PATCH 252/367] fix uuid comparison

---
 RemoteDaemon/Transport/Tor.hs | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs
index 6149df3767..220a3616d2 100644
--- a/RemoteDaemon/Transport/Tor.hs
+++ b/RemoteDaemon/Transport/Tor.hs
@@ -133,14 +133,16 @@ transport (RemoteRepo r _) url@(RemoteURI uri) th ichan ochan =
 		res <- runNetProto conn $
 			P2P.auth myuuid authtoken
 		case res of
-			Right (Just theiruuid)
-				| getUncachedUUID r == theiruuid -> do
-					send (CONNECTED url)
-					status <- handlecontrol
-						`race` handlepeer conn
-					send (DISCONNECTED url)
-					return $ either id id status
-				| otherwise -> return ConnectionStopping
+			Right (Just theiruuid) -> do
+				expecteduuid <- liftAnnex th $ getRepoUUID r
+				if expecteduuid == theiruuid
+					then do
+						send (CONNECTED url)
+						status <- handlecontrol
+							`race` handlepeer conn
+						send (DISCONNECTED url)
+						return $ either id id status
+					else return ConnectionStopping
 			_ -> return ConnectionClosed
 	
 	send msg = atomically $ writeTChan ochan msg

From b65780778a4061b478cc94b0f67bd3f845992986 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 9 Dec 2016 17:07:32 -0400
Subject: [PATCH 253/367] update

---
 doc/todo/tor.mdwn | 1 -
 1 file changed, 1 deletion(-)

diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index 9bbc257ad4..01fa2635ac 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -10,7 +10,6 @@ Current todo list:
   "Connection reset by peer"
 * Think about locking some more. What happens if the connection to the peer
   is dropped while we think we're locking content there from being dropped?
-* test remotedaemon's change detection
 
 Eventually:
 

From fa1b3a19f90d475b5892be54b602d5dc780e5201 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 9 Dec 2016 17:11:16 -0400
Subject: [PATCH 254/367] hang up connection after relaying

Seems that git upload-pack outputs a "ONCDN " that is not read by the
remote git receive-pack. This fixes:

[2016-12-09 17:08:32.77159731] P2P > ERROR protocol parse error: "ONCDN "
---
 P2P/Protocol.hs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/P2P/Protocol.hs b/P2P/Protocol.hs
index c3c362f37d..135409e262 100644
--- a/P2P/Protocol.hs
+++ b/P2P/Protocol.hs
@@ -387,7 +387,10 @@ serveAuthed myuuid = void $ serverLoop handler
 		return ServerContinue
 	handler (CONNECT service) = do
 		net $ relayService service
-		return ServerContinue
+		-- After connecting to git, there may be unconsumed data
+		-- from the git processes hanging around (even if they
+		-- exited successfully), so stop serving this connection.
+		return $ ServerGot ()
 	handler NOTIFYCHANGE = do
 		refs <- local waitRefChange
 		net $ sendMessage (CHANGED refs)

From d07a85e74e14b33d0124d5f577b35bd4bd765d07 Mon Sep 17 00:00:00 2001
From: alpernebbi <alpernebbi@web>
Date: Sat, 10 Dec 2016 07:36:04 +0000
Subject: [PATCH 255/367] Added a comment

---
 ...comment_3_7342e29b0d2225abc5800638e3b377ed._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/git-annex_fromkey_barfs_on_utf-8_input/comment_3_7342e29b0d2225abc5800638e3b377ed._comment

diff --git a/doc/bugs/git-annex_fromkey_barfs_on_utf-8_input/comment_3_7342e29b0d2225abc5800638e3b377ed._comment b/doc/bugs/git-annex_fromkey_barfs_on_utf-8_input/comment_3_7342e29b0d2225abc5800638e3b377ed._comment
new file mode 100644
index 0000000000..5011aa1fa5
--- /dev/null
+++ b/doc/bugs/git-annex_fromkey_barfs_on_utf-8_input/comment_3_7342e29b0d2225abc5800638e3b377ed._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="alpernebbi"
+ avatar="http://cdn.libravatar.org/avatar/daf2abb14f39e28ad75d5f9a03fcd106"
+ subject="comment 3"
+ date="2016-12-10T07:36:04Z"
+ content="""
+I experienced all these with the [linux standalone](https://git-annex.branchable.com/install/Linux_standalone/) from this site as well.
+
+However, I couldn't reproduce them in a Debian unstable chroot where I installed the `git-annex` package from their repos.
+"""]]

From efa71e70a43f17434bbd5031cd8973a8b92598d1 Mon Sep 17 00:00:00 2001
From: "0xloem@0bd8a79a57e4f0dcade8fc81d162c37eae4d6730" <0xloem@web>
Date: Sat, 10 Dec 2016 12:31:32 +0000
Subject: [PATCH 256/367] Added a comment: Corrupt Links Produced, Significant
 Data Loss

---
 ...nt_1_b76704adf6b6aa441a35bf9458d3950d._comment | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_1_b76704adf6b6aa441a35bf9458d3950d._comment

diff --git a/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_1_b76704adf6b6aa441a35bf9458d3950d._comment b/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_1_b76704adf6b6aa441a35bf9458d3950d._comment
new file mode 100644
index 0000000000..3fe7af8dd3
--- /dev/null
+++ b/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_1_b76704adf6b6aa441a35bf9458d3950d._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="0xloem@0bd8a79a57e4f0dcade8fc81d162c37eae4d6730"
+ nickname="0xloem"
+ avatar="http://cdn.libravatar.org/avatar/b8c087f7c5e6a9358748f0727c077f3b"
+ subject="Corrupt Links Produced, Significant Data Loss"
+ date="2016-12-10T12:31:31Z"
+ content="""
+Additionally, I added a mess of files with this release of git-annex, and of the 200 files added, three of the links produced were corrupt.  I'm still searching for where these files have gone to recover the data.
+
+The files look like this in `ls -l`, they were bup files:
+
+    lrwxrwxrwx 1 user user 338 Jun 17 22:36 bup.git/objects/pack/pack-47b493a3bbbd22200d2b390c277e49ce713243cc.pack -> *??:?;J?????????
+    lrwxrwxrwx 1 user user 336 Jun 17 21:41 bup.git/objects/pack/pack-4d202b3929b187d4acaf1602526e7344eef1edc8.pack -> ?p???GWj??????ܥ??{b?#???>C??%??????~à???/hjT;?p??d?8??oyE?K?)6?uL+??h??&???SB}?'s??֫{?>^i?&?f??^{ш??aD??t4?C?sBTk>d6H???5h3?ڋ6fAa??=?r????j?????a8K??????????B?~????I͕?T7?Y??=???b?7C???鋤??8???\"?????#???M?????}z?A??9?C>?-?GD??7?ј;'P?H??ɑ??Zr?/U???W?G??3@\"??Ȧ?z?h???U??Ԇ???R??u??I????62??>@??@?a??x???}?????)d?G;(???m_?^3?????T
+    lrwxrwxrwx 1 user user 332 Jul 20 07:32 bup.git/objects/pack/pack-5328381f3b023c1356581c22d1e74d4eda0b46a3.idx -> c??'w??????????m?q#?ٱCO??o????ʃ?Ʃڌ??[???Ѐ??*?;.?c?N?0?????D$ o?r????8BGn?96gY?B?Z1?=???{??z?71????!aG?>?u)???i\?G[???:?Kk??%??.mu???n???K??ǚ????q&Z-?E???]??/?6???}
+"""]]

From bbc5e4f4f3db354faf9ea1d30f5db5de9931c6b6 Mon Sep 17 00:00:00 2001
From: alpernebbi <alpernebbi@web>
Date: Sat, 10 Dec 2016 13:08:58 +0000
Subject: [PATCH 257/367] Added a comment: Patch to fix aws head build issue

---
 ..._d48bc2b3eb48c2a3a4d8608803913000._comment | 149 ++++++++++++++++++
 1 file changed, 149 insertions(+)
 create mode 100644 doc/bugs/Build_with_aws_head_fails/comment_1_d48bc2b3eb48c2a3a4d8608803913000._comment

diff --git a/doc/bugs/Build_with_aws_head_fails/comment_1_d48bc2b3eb48c2a3a4d8608803913000._comment b/doc/bugs/Build_with_aws_head_fails/comment_1_d48bc2b3eb48c2a3a4d8608803913000._comment
new file mode 100644
index 0000000000..536c1569dc
--- /dev/null
+++ b/doc/bugs/Build_with_aws_head_fails/comment_1_d48bc2b3eb48c2a3a4d8608803913000._comment
@@ -0,0 +1,149 @@
+[[!comment format=mdwn
+ username="alpernebbi"
+ avatar="http://cdn.libravatar.org/avatar/daf2abb14f39e28ad75d5f9a03fcd106"
+ subject="Patch to fix aws head build issue"
+ date="2016-12-10T13:08:58Z"
+ content="""
+I think I fixed this. I'm attaching the output of `git format-patch origin/master`.
+
+In an Arch Linux chroot with their [PKGBUILD](https://git.archlinux.org/svntogit/community.git/tree/trunk/PKGBUILD?h=packages/git-annex) (with a small modification to apply the patch), `haskell-http-client 0.5.3.3-1` and `http-conduit 2.2.3-5` both the build and the tests are successful.
+It's also successful in a Debian Sid chroot, where `sudo apt build-dep git-annex` gives me `libghc-http-client-dev 0.4.31.1-3+b2`, `libghc-http-conduit-dev 2.1.11-3+b2`.
+
+### Patch
+
+[[!format patch \"\"\"
+From 2ce09420aa8f3d916c6562abea4ed8911a186902 Mon Sep 17 00:00:00 2001
+From: Alper Nebi Yasak <alpernebiyasak@gmail.com>
+Date: Sat, 10 Dec 2016 15:24:27 +0300
+Subject: [PATCH] Remove http-conduit (<2.2.0) constraint
+
+Since https://github.com/aristidb/aws/issues/206 is resolved, this
+constraint is no longer necessary. However, http-conduit (>=2.2.0)
+requires http-client (>=0.5.0) which introduces some breaking changes.
+This commit also implements those changes depending on the version.
+Fixes: https://git-annex.branchable.com/bugs/Build_with_aws_head_fails/
+
+Signed-off-by: Alper Nebi Yasak <alpernebiyasak@gmail.com>
+---
+ Remote/S3.hs     |  8 +++++++-
+ Remote/WebDAV.hs | 17 +++++++++++++++++
+ Utility/Url.hs   |  8 ++++++++
+ git-annex.cabal  |  3 +--
+ 4 files changed, 33 insertions(+), 3 deletions(-)
+
+diff --git a/Remote/S3.hs b/Remote/S3.hs
+index 4c1bd57..9563b5a 100644
+--- a/Remote/S3.hs
++++ b/Remote/S3.hs
+@@ -49,6 +49,12 @@ import Annex.Content
+ import Annex.Url (withUrlOptions)
+ import Utility.Url (checkBoth, managerSettings, closeManager)
+ 
++#if MIN_VERSION_http_client(0,5,0)
++import Network.HTTP.Client (responseTimeoutNone)
++#else
++responseTimeoutNone = Nothing
++#endif
++
+ type BucketName = String
+ 
+ remote :: RemoteType
+@@ -430,7 +436,7 @@ withS3HandleMaybe c gc u a = do
+   where
+ 	s3cfg = s3Configuration c
+ 	httpcfg = managerSettings
+-		{ managerResponseTimeout = Nothing }
++		{ managerResponseTimeout = responseTimeoutNone }
+ 
+ s3Configuration :: RemoteConfig -> S3.S3Configuration AWS.NormalQuery
+ s3Configuration c = cfg
+diff --git a/Remote/WebDAV.hs b/Remote/WebDAV.hs
+index 19dbaa8..14947f1 100644
+--- a/Remote/WebDAV.hs
++++ b/Remote/WebDAV.hs
+@@ -5,6 +5,7 @@
+  - Licensed under the GNU GPL version 3 or higher.
+  -}
+ 
++{-# LANGUAGE CPP #-}
+ {-# LANGUAGE ScopedTypeVariables #-}
+ 
+ module Remote.WebDAV (remote, davCreds, configUrl) where
+@@ -34,6 +35,10 @@ import Utility.Url (URLString, matchStatusCodeException)
+ import Annex.UUID
+ import Remote.WebDAV.DavLocation
+ 
++#if MIN_VERSION_http_client(0,5,0)
++import Network.HTTP.Client (HttpExceptionContent(..), responseStatus)
++#endif
++
+ remote :: RemoteType
+ remote = RemoteType {
+ 	typename = \"webdav\",
+@@ -302,6 +307,17 @@ goDAV (DavHandle ctx user pass _) a = choke $ run $ prettifyExceptions $ do
+ {- Catch StatusCodeException and trim it to only the statusMessage part,
+  - eliminating a lot of noise, which can include the whole request that
+  - failed. The rethrown exception is no longer a StatusCodeException. -}
++#if MIN_VERSION_http_client(0,5,0)
++prettifyExceptions :: DAVT IO a -> DAVT IO a
++prettifyExceptions a = catchJust (matchStatusCodeException (const True)) a go
++  where
++	go (HttpExceptionRequest _ (StatusCodeException response message)) = error $ unwords
++		[ \"DAV failure:\"
++		, show (responseStatus response)
++		, show (message)
++		]
++	go e = throwM e
++#else
+ prettifyExceptions :: DAVT IO a -> DAVT IO a
+ prettifyExceptions a = catchJust (matchStatusCodeException (const True)) a go
+   where
+@@ -311,6 +327,7 @@ prettifyExceptions a = catchJust (matchStatusCodeException (const True)) a go
+ 		, show (statusMessage status)
+ 		]
+ 	go e = throwM e
++#endif
+ 
+ prepDAV :: DavUser -> DavPass -> DAVT IO ()
+ prepDAV user pass = do
+diff --git a/Utility/Url.hs b/Utility/Url.hs
+index 9b68871..d0e1b37 100644
+--- a/Utility/Url.hs
++++ b/Utility/Url.hs
+@@ -350,8 +350,16 @@ hUserAgent = \"User-Agent\"
+  -
+  - > catchJust (matchStatusCodeException (== notFound404))
+  -}
++#if MIN_VERSION_http_client(0,5,0)
++matchStatusCodeException :: (Status -> Bool) -> HttpException -> Maybe HttpException
++matchStatusCodeException want e@(HttpExceptionRequest _ (StatusCodeException r _))
++	| want (responseStatus r) = Just e
++	| otherwise = Nothing
++matchStatusCodeException _ _ = Nothing
++#else
+ matchStatusCodeException :: (Status -> Bool) -> HttpException -> Maybe HttpException
+ matchStatusCodeException want e@(StatusCodeException s _ _)
+ 	| want s = Just e
+ 	| otherwise = Nothing
+ matchStatusCodeException _ _ = Nothing
++#endif
+diff --git a/git-annex.cabal b/git-annex.cabal
+index ec54a14..83d45a1 100644
+--- a/git-annex.cabal
++++ b/git-annex.cabal
+@@ -357,8 +357,7 @@ Executable git-annex
+    resourcet,
+    http-client,
+    http-types,
+-   -- Old version needed due to https://github.com/aristidb/aws/issues/206
+-   http-conduit (<2.2.0),
++   http-conduit,
+    time,
+    old-locale,
+    esqueleto,
+-- 
+2.7.4
+
+\"\"\"]]
+
+"""]]

From 93a22a1c972d58b17bc0e23a4b5c9768310f943c Mon Sep 17 00:00:00 2001
From: Alper Nebi Yasak <alpernebiyasak@gmail.com>
Date: Sat, 10 Dec 2016 15:24:27 +0300
Subject: [PATCH 258/367] Remove http-conduit (<2.2.0) constraint

Since https://github.com/aristidb/aws/issues/206 is resolved, this
constraint is no longer necessary. However, http-conduit (>=2.2.0)
requires http-client (>=0.5.0) which introduces some breaking changes.
This commit also implements those changes depending on the version.
Fixes: https://git-annex.branchable.com/bugs/Build_with_aws_head_fails/

Signed-off-by: Alper Nebi Yasak <alpernebiyasak@gmail.com>
---
 Remote/S3.hs     |  8 +++++++-
 Remote/WebDAV.hs | 17 +++++++++++++++++
 Utility/Url.hs   |  8 ++++++++
 git-annex.cabal  |  3 +--
 4 files changed, 33 insertions(+), 3 deletions(-)

diff --git a/Remote/S3.hs b/Remote/S3.hs
index 4c1bd5784f..9563b5a0fa 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -49,6 +49,12 @@ import Annex.Content
 import Annex.Url (withUrlOptions)
 import Utility.Url (checkBoth, managerSettings, closeManager)
 
+#if MIN_VERSION_http_client(0,5,0)
+import Network.HTTP.Client (responseTimeoutNone)
+#else
+responseTimeoutNone = Nothing
+#endif
+
 type BucketName = String
 
 remote :: RemoteType
@@ -430,7 +436,7 @@ withS3HandleMaybe c gc u a = do
   where
 	s3cfg = s3Configuration c
 	httpcfg = managerSettings
-		{ managerResponseTimeout = Nothing }
+		{ managerResponseTimeout = responseTimeoutNone }
 
 s3Configuration :: RemoteConfig -> S3.S3Configuration AWS.NormalQuery
 s3Configuration c = cfg
diff --git a/Remote/WebDAV.hs b/Remote/WebDAV.hs
index 19dbaa8af5..14947f1e9b 100644
--- a/Remote/WebDAV.hs
+++ b/Remote/WebDAV.hs
@@ -5,6 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
+{-# LANGUAGE CPP #-}
 {-# LANGUAGE ScopedTypeVariables #-}
 
 module Remote.WebDAV (remote, davCreds, configUrl) where
@@ -34,6 +35,10 @@ import Utility.Url (URLString, matchStatusCodeException)
 import Annex.UUID
 import Remote.WebDAV.DavLocation
 
+#if MIN_VERSION_http_client(0,5,0)
+import Network.HTTP.Client (HttpExceptionContent(..), responseStatus)
+#endif
+
 remote :: RemoteType
 remote = RemoteType {
 	typename = "webdav",
@@ -302,6 +307,17 @@ goDAV (DavHandle ctx user pass _) a = choke $ run $ prettifyExceptions $ do
 {- Catch StatusCodeException and trim it to only the statusMessage part,
  - eliminating a lot of noise, which can include the whole request that
  - failed. The rethrown exception is no longer a StatusCodeException. -}
+#if MIN_VERSION_http_client(0,5,0)
+prettifyExceptions :: DAVT IO a -> DAVT IO a
+prettifyExceptions a = catchJust (matchStatusCodeException (const True)) a go
+  where
+	go (HttpExceptionRequest _ (StatusCodeException response message)) = error $ unwords
+		[ "DAV failure:"
+		, show (responseStatus response)
+		, show (message)
+		]
+	go e = throwM e
+#else
 prettifyExceptions :: DAVT IO a -> DAVT IO a
 prettifyExceptions a = catchJust (matchStatusCodeException (const True)) a go
   where
@@ -311,6 +327,7 @@ prettifyExceptions a = catchJust (matchStatusCodeException (const True)) a go
 		, show (statusMessage status)
 		]
 	go e = throwM e
+#endif
 
 prepDAV :: DavUser -> DavPass -> DAVT IO ()
 prepDAV user pass = do
diff --git a/Utility/Url.hs b/Utility/Url.hs
index 9b68871dd9..d0e1b3739c 100644
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -350,8 +350,16 @@ hUserAgent = "User-Agent"
  -
  - > catchJust (matchStatusCodeException (== notFound404))
  -}
+#if MIN_VERSION_http_client(0,5,0)
+matchStatusCodeException :: (Status -> Bool) -> HttpException -> Maybe HttpException
+matchStatusCodeException want e@(HttpExceptionRequest _ (StatusCodeException r _))
+	| want (responseStatus r) = Just e
+	| otherwise = Nothing
+matchStatusCodeException _ _ = Nothing
+#else
 matchStatusCodeException :: (Status -> Bool) -> HttpException -> Maybe HttpException
 matchStatusCodeException want e@(StatusCodeException s _ _)
 	| want s = Just e
 	| otherwise = Nothing
 matchStatusCodeException _ _ = Nothing
+#endif
diff --git a/git-annex.cabal b/git-annex.cabal
index ec54a146d9..83d45a1d94 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -357,8 +357,7 @@ Executable git-annex
    resourcet,
    http-client,
    http-types,
-   -- Old version needed due to https://github.com/aristidb/aws/issues/206
-   http-conduit (<2.2.0),
+   http-conduit,
    time,
    old-locale,
    esqueleto,

From 749623df86e5417b235e6dd29376d1726e4a542f Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 10 Dec 2016 10:46:51 -0400
Subject: [PATCH 259/367] fixed

---
 CHANGELOG                               | 2 ++
 doc/bugs/Build_with_aws_head_fails.mdwn | 1 +
 2 files changed, 3 insertions(+)

diff --git a/CHANGELOG b/CHANGELOG
index b741f29b95..095d1e7064 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -24,6 +24,8 @@ git-annex (6.20161119) UNRELEASED; urgency=medium
   * add: Stage modified non-large files when running in indirect mode. 
     (This was already done in v6 mode and direct mode.)
   * git-annex-shell, remotedaemon, git remote: Fix some memory DOS attacks.
+  * Fix build with http-client 0.5.
+    Thanks, Alper Nebi Yasak.
 
  -- Joey Hess <id@joeyh.name>  Mon, 21 Nov 2016 11:27:50 -0400
 
diff --git a/doc/bugs/Build_with_aws_head_fails.mdwn b/doc/bugs/Build_with_aws_head_fails.mdwn
index 9937d6c35b..a96dce0ad9 100644
--- a/doc/bugs/Build_with_aws_head_fails.mdwn
+++ b/doc/bugs/Build_with_aws_head_fails.mdwn
@@ -46,3 +46,4 @@ ExitFailure 1
 ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
 Yes :)
 
+> [[done]] via the nice patch! --[[Joey]]

From c2d9f4397ebf50fc16a34148cdabe4022db778a9 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 10 Dec 2016 11:04:11 -0400
Subject: [PATCH 260/367] fix typo

---
 debian/control | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/control b/debian/control
index f5a9de8408..58af2306cc 100644
--- a/debian/control
+++ b/debian/control
@@ -65,7 +65,7 @@ Build-Depends:
 	libghc-xml-types-dev,
 	libghc-async-dev,
 	libghc-monad-logger-dev,
-	ligghc-free-dev,
+	libghc-free-dev,
 	libghc-feed-dev (>= 0.3.9.2),
 	libghc-regex-tdfa-dev,
 	libghc-tasty-dev (>= 0.7),

From 16c6333f09ec214c9612bb638e58e19d6bddca99 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 10 Dec 2016 11:12:18 -0400
Subject: [PATCH 261/367] fix build with old ghc

---
 P2P/Annex.hs | 2 +-
 P2P/IO.hs    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/P2P/Annex.hs b/P2P/Annex.hs
index b3db7513c0..9971762f59 100644
--- a/P2P/Annex.hs
+++ b/P2P/Annex.hs
@@ -34,7 +34,7 @@ runFullProto :: RunMode -> P2PConnection -> Proto a -> Annex (Either String a)
 runFullProto runmode conn = go
   where
 	go :: RunProto Annex
-	go (Pure v) = pure (Right v)
+	go (Pure v) = return (Right v)
 	go (Free (Net n)) = runNet conn go n
 	go (Free (Local l)) = runLocal runmode go l
 
diff --git a/P2P/IO.hs b/P2P/IO.hs
index bb93528a65..3e0999775f 100644
--- a/P2P/IO.hs
+++ b/P2P/IO.hs
@@ -85,7 +85,7 @@ runNetProto :: P2PConnection -> Proto a -> IO (Either String a)
 runNetProto conn = go
   where
 	go :: RunProto IO
-	go (Pure v) = pure (Right v)
+	go (Pure v) = return (Right v)
 	go (Free (Net n)) = runNet conn go n
 	go (Free (Local _)) = return (Left "unexpected annex operation attempted")
 

From d44694cdd13d947fb1076ba69bb833b00ad32fc5 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 10 Dec 2016 11:31:55 -0400
Subject: [PATCH 262/367] followup

---
 ..._3469bfd3ba5e7935f3350f0bd78a0c94._comment | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_2_3469bfd3ba5e7935f3350f0bd78a0c94._comment

diff --git a/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_2_3469bfd3ba5e7935f3350f0bd78a0c94._comment b/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_2_3469bfd3ba5e7935f3350f0bd78a0c94._comment
new file mode 100644
index 0000000000..52932cd5e9
--- /dev/null
+++ b/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_2_3469bfd3ba5e7935f3350f0bd78a0c94._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-12-10T15:13:44Z"
+ content="""
+I think that the "x86-32, for ancient kernels" build should avoid this
+problem. <http://git-annex.branchable.com/install/Linux_standalone/>
+
+It's very surprising if this lead to symlinks being created that apparently
+contain garbage in their link targets. Perhaps glibc is failing in a way
+with the old kernel that leads to memory corruption? I have asked the GHC
+developers if that could be the case in
+<https://ghc.haskell.org/trac/ghc/ticket/12865>
+
+I hope that the content of your files is in fact somewhere under
+`.git/annex/objects/` -- look around, and with some luck, you may find it.
+Unfortunately, the information about, which object file goes with which
+working tree has apparently been lost. (Also, you might check if these
+symlinks have been staged in git; it's possible though unlikely that the
+correct link target got staged in git.)
+
+I have filed a bug on Debian's ghc to get them to fast-track getting the
+patch into ghc. <https://bugs.debian.org/847677>
+"""]]

From effd50d918d92e4adff1aa69da735704d76a9309 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 10 Dec 2016 11:32:05 -0400
Subject: [PATCH 263/367] fix build with old stm

Old stm lacks isFullTMQueue.

To avoid needing to update stm on the Android autobuilder, I switched to
a TBMQueue. It never needs to be closed, but the overhead is minimal.
---
 RemoteDaemon/Transport/Tor.hs | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs
index 220a3616d2..61e1189a54 100644
--- a/RemoteDaemon/Transport/Tor.hs
+++ b/RemoteDaemon/Transport/Tor.hs
@@ -31,6 +31,7 @@ import System.PosixCompat.User
 import Control.Concurrent
 import System.Log.Logger (debugM)
 import Control.Concurrent.STM
+import Control.Concurrent.STM.TBMQueue
 import Control.Concurrent.Async
 import qualified Network.Socket as S
 
@@ -39,7 +40,7 @@ server :: TransportHandle -> IO ()
 server th@(TransportHandle (LocalRepo r) _) = do
 	u <- liftAnnex th getUUID
 
-	q <- newTBQueueIO maxConnections
+	q <- newTBMQueueIO maxConnections
 	replicateM_ maxConnections $
 		forkIO $ forever $ serveClient th u r q
 
@@ -59,10 +60,10 @@ server th@(TransportHandle (LocalRepo r) _) = do
 	forever $ do
 		(conn, _) <- S.accept soc
 		h <- setupHandle conn
-		ok <- atomically $ ifM (isFullTBQueue q)
+		ok <- atomically $ ifM (isFullTBMQueue q)
 			( return False
 			, do
-				writeTBQueue q h
+				writeTBMQueue q h
 				return True
 			)
 		unless ok $ do
@@ -73,19 +74,21 @@ server th@(TransportHandle (LocalRepo r) _) = do
 maxConnections :: Int
 maxConnections = 100
 
-serveClient :: TransportHandle -> UUID -> Repo -> TBQueue Handle -> IO ()
+serveClient :: TransportHandle -> UUID -> Repo -> TBMQueue Handle -> IO ()
 serveClient th u r q = bracket setup cleanup start
   where
 	setup = do
-		h <- atomically $ readTBQueue q
+		h <- atomically $ readTBMQueue q
 		debugM "remotedaemon" "serving a Tor connection"
 		return h
 	
-	cleanup h = do
+	cleanup Nothing = return ()
+	cleanup (Just h) = do
 		debugM "remotedaemon" "done with Tor connection"
 		hClose h
 
-	start h = do
+	start Nothing = return ()
+	start (Just h) = do
 		-- Avoid doing any work in the liftAnnex, since only one
 		-- can run at a time.
 		st <- liftAnnex th dupState

From b72352e1b1473eafd57596f63a35e3d399fcce33 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 10 Dec 2016 11:41:38 -0400
Subject: [PATCH 264/367] fix build warning

---
 Remote/S3.hs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Remote/S3.hs b/Remote/S3.hs
index 9563b5a0fa..2b7c58e6fc 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -52,6 +52,7 @@ import Utility.Url (checkBoth, managerSettings, closeManager)
 #if MIN_VERSION_http_client(0,5,0)
 import Network.HTTP.Client (responseTimeoutNone)
 #else
+responseTimeoutNone :: Maybe Int
 responseTimeoutNone = Nothing
 #endif
 

From fd1f127996dcf4586d435faacbdbf9a62ec153ea Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 10 Dec 2016 11:45:46 -0400
Subject: [PATCH 265/367] warning about old kernels

---
 doc/install/Linux_standalone.mdwn | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/doc/install/Linux_standalone.mdwn b/doc/install/Linux_standalone.mdwn
index 223c8189b3..5bad5f930b 100644
--- a/doc/install/Linux_standalone.mdwn
+++ b/doc/install/Linux_standalone.mdwn
@@ -5,6 +5,13 @@ prebuilt tarball of the most recent release.
 This tarball should work on most Linux systems. It has basically no
 dependencies and is self-contained.
 
+**Warning** Linux kernels older than 4.5 may not run this build correctly, and
+<a href="http://git-annex.branchable.com/bugs/unable_to_decommit_memory__58___Invalid_argument/">data loss has been reported</a>
+using this build with the old kernel. We are working to resolve this
+problem, and if you use such a kernel, we recommend running
+`git annex test` to check git-annex works correctly, and/or 
+using the build for ancient kernels.
+
 * x86-32: [download tarball](https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-i386.tar.gz)
 * x86-64: [download tarball](https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-amd64.tar.gz)
 * x86-32, for ancient kernels: [download tarball](https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-i386-ancient.tar.gz)

From dd715b5993ad5982a6c19b1e633c8041f804b881 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 10 Dec 2016 11:55:56 -0400
Subject: [PATCH 266/367] improve hask to work with TH for Free monads

---
 Build/EvilSplicer.hs | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/Build/EvilSplicer.hs b/Build/EvilSplicer.hs
index e685b1e443..a24f265e22 100644
--- a/Build/EvilSplicer.hs
+++ b/Build/EvilSplicer.hs
@@ -474,7 +474,7 @@ mangleCode = flip_colon
 	 -
 	 - To fix, we could just put a semicolon at the start of every line
 	 - containing " -> " ... Except that lambdas also contain that.
-	 - But we can get around that: GHC outputs lambas like this:
+	 - But we can get around that: GHC outputs lambdas like this:
 	 -
 	 - \ foo
 	 -   -> bar
@@ -486,6 +486,15 @@ mangleCode = flip_colon
 	 - So, we can put the semicolon at the start of every line
 	 - containing " -> " unless there's a "\ " first, or it's
 	 - all whitespace up until it.
+	 -
+	 - Except.. type signatures also contain " -> " sometimes starting
+	 - a line:
+	 -
+	 - forall foo =>
+	 -   Foo  ->
+	 -
+	 - To detect and avoid these, note that ghc puts 2 spaces
+	 - before the "->"
 	 -}
 	case_layout = parsecAndReplace $ do
 		void newline
@@ -497,7 +506,9 @@ mangleCode = flip_colon
 				then unexpected "lambda expression"
 				else if null prefix
 					then unexpected "second line of lambda"
-					else return $ "\n" ++ indent1 ++ "; " ++ prefix ++ " -> "
+					else if "  " `isSuffixOf` prefix
+						then unexpected "probably type signature"
+						else return $ "\n" ++ indent1 ++ "; " ++ prefix ++ " -> "
 	{- Sometimes cases themselves span multiple lines:
 	 -
 	 - Nothing

From a88b5e77865255d7904dfe20334581d662f500c3 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 10 Dec 2016 12:13:41 -0400
Subject: [PATCH 267/367] better fix

---
 Build/EvilSplicer.hs | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/Build/EvilSplicer.hs b/Build/EvilSplicer.hs
index a24f265e22..4259717230 100644
--- a/Build/EvilSplicer.hs
+++ b/Build/EvilSplicer.hs
@@ -491,12 +491,13 @@ mangleCode = flip_colon
 	 - a line:
 	 -
 	 - forall foo =>
-	 -   Foo  ->
+	 -   Foo ->
 	 -
-	 - To detect and avoid these, note that ghc puts 2 spaces
-	 - before the "->"
+	 - To avoid breaking these, look for the => on the previous line.
 	 -}
 	case_layout = parsecAndReplace $ do
+		void newline
+		lastline <- restOfLine
 		void newline
 		indent1 <- many1 $ char ' '
 		prefix <- manyTill (noneOf "\n") (try (string "-> "))
@@ -506,7 +507,7 @@ mangleCode = flip_colon
 				then unexpected "lambda expression"
 				else if null prefix
 					then unexpected "second line of lambda"
-					else if "  " `isSuffixOf` prefix
+					else if "=>" `isSuffixOf` lastline
 						then unexpected "probably type signature"
 						else return $ "\n" ++ indent1 ++ "; " ++ prefix ++ " -> "
 	{- Sometimes cases themselves span multiple lines:

From 73a79147b13c994982b490e076e8af7b33cd4207 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 10 Dec 2016 12:23:18 -0400
Subject: [PATCH 268/367] releasing package git-annex version 6.20161210

---
 CHANGELOG       | 4 ++--
 git-annex.cabal | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index 095d1e7064..bb54989e86 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,4 +1,4 @@
-git-annex (6.20161119) UNRELEASED; urgency=medium
+git-annex (6.20161210) unstable; urgency=medium
 
   * enable-tor: New command, enables tor hidden service for P2P syncing.
   * p2p: New command, allows linking repositories using a P2P network.
@@ -27,7 +27,7 @@ git-annex (6.20161119) UNRELEASED; urgency=medium
   * Fix build with http-client 0.5.
     Thanks, Alper Nebi Yasak.
 
- -- Joey Hess <id@joeyh.name>  Mon, 21 Nov 2016 11:27:50 -0400
+ -- Joey Hess <id@joeyh.name>  Sat, 10 Dec 2016 11:56:25 -0400
 
 git-annex (6.20161118) unstable; urgency=medium
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 83d45a1d94..6b81424ab5 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 6.20161118
+Version: 6.20161210
 Cabal-Version: >= 1.8
 License: GPL-3
 Maintainer: Joey Hess <id@joeyh.name>

From 474caa1885c51562ab9e4814f1a24d5c4e695aca Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 10 Dec 2016 12:23:34 -0400
Subject: [PATCH 269/367] add news item for git-annex 6.20161210

---
 doc/news/version_6.20161012.mdwn | 30 ------------------------------
 doc/news/version_6.20161210.mdwn | 28 ++++++++++++++++++++++++++++
 2 files changed, 28 insertions(+), 30 deletions(-)
 delete mode 100644 doc/news/version_6.20161012.mdwn
 create mode 100644 doc/news/version_6.20161210.mdwn

diff --git a/doc/news/version_6.20161012.mdwn b/doc/news/version_6.20161012.mdwn
deleted file mode 100644
index a6cb017800..0000000000
--- a/doc/news/version_6.20161012.mdwn
+++ /dev/null
@@ -1,30 +0,0 @@
-git-annex 6.20161012 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Optimisations to time it takes git-annex to walk working tree and find
-     files to work on. Sped up by around 18%.
-   * Optimisations to git-annex branch query and setting, avoiding repeated
-     copies of the environment. Speeds up commands like
-     "git-annex find --in remote" by over 50%.
-   * Optimised git-annex branch log file timestamp parsing.
-   * Add "total-size" field to --json-progress output.
-   * Make --json-progress output be shown even when the size of a object
-     is not known.
-   * Multiple external special remote processes for the same remote will be
-     started as needed when using -J. This should not beak any existing
-     external special remotes, because running multiple git-annex commands
-     at the same time could already start multiple processes for the same
-     external special remotes.
-   * Linux standalone: Include locale files in the bundle, and generate
-     locale definition files for the locales in use when starting runshell.
-     (Currently only done for utf-8 locales.)
-   * Avoid using a lot of memory when large objects are present in the git
-     repository and have to be checked to see if they are a pointed to an
-     annexed file. Cases where such memory use could occur included, but
-     were not limited to:
-     - git commit -a of a large unlocked file (in v5 mode)
-     - git-annex adjust when a large file was checked into git directly
-   * When auto-upgrading a v3 remote, avoid upgrading to version 6,
-     instead keep it at version 5.
-   * Support using v3 repositories without upgrading them to v5.
-   * sync: Fix bug in adjusted branch merging that could cause recently
-     added files to be lost when updating the adjusted branch."""]]
\ No newline at end of file
diff --git a/doc/news/version_6.20161210.mdwn b/doc/news/version_6.20161210.mdwn
new file mode 100644
index 0000000000..5c3584d303
--- /dev/null
+++ b/doc/news/version_6.20161210.mdwn
@@ -0,0 +1,28 @@
+git-annex 6.20161210 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * enable-tor: New command, enables tor hidden service for P2P syncing.
+   * p2p: New command, allows linking repositories using a P2P network.
+   * remotedaemon: Serve tor hidden service.
+   * Added git-remote-tor-annex, which allows git pull and push to the tor
+     hidden service.
+   * remotedaemon: Fork to background by default. Added --foreground switch
+     to enable old behavior.
+   * addurl: Fix bug in checking annex.largefiles expressions using
+     largerthan, mimetype, and smallerthan; the first two always failed
+     to match, and the latter always matched.
+   * Relicense 5 source files that are not part of the webapp from AGPL to GPL.
+   * map: Run xdot if it's available in PATH. On OSX, the dot command
+     does not support graphical display, while xdot does.
+   * Debian: xdot is a better interactive viewer than dot, so Suggest
+     xdot, rather than graphviz.
+   * rmurl: Multiple pairs of files and urls can be provided on the
+     command line.
+   * rmurl: Added --batch mode.
+   * fromkey: Accept multiple pairs of files and keys.
+     Thanks, Daniel Brooks.
+   * rekey: Added --batch mode.
+   * add: Stage modified non-large files when running in indirect mode.
+     (This was already done in v6 mode and direct mode.)
+   * git-annex-shell, remotedaemon, git remote: Fix some memory DOS attacks.
+   * Fix build with http-client 0.5.
+     Thanks, Alper Nebi Yasak."""]]
\ No newline at end of file

From f4b7caaa61b36e8024324632e81d15128e2360ea Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 10 Dec 2016 12:29:29 -0400
Subject: [PATCH 270/367] fix some more

---
 Build/EvilSplicer.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Build/EvilSplicer.hs b/Build/EvilSplicer.hs
index 4259717230..9a9dffa420 100644
--- a/Build/EvilSplicer.hs
+++ b/Build/EvilSplicer.hs
@@ -509,7 +509,7 @@ mangleCode = flip_colon
 					then unexpected "second line of lambda"
 					else if "=>" `isSuffixOf` lastline
 						then unexpected "probably type signature"
-						else return $ "\n" ++ indent1 ++ "; " ++ prefix ++ " -> "
+						else return $ "\n" ++ lastline ++ "\n" ++ indent1 ++ "; " ++ prefix ++ " -> "
 	{- Sometimes cases themselves span multiple lines:
 	 -
 	 - Nothing

From 5fb7b87bf09de42d84f62a6b309bf1fd8701cf53 Mon Sep 17 00:00:00 2001
From: "0xloem@0bd8a79a57e4f0dcade8fc81d162c37eae4d6730" <0xloem@web>
Date: Sun, 11 Dec 2016 00:26:42 +0000
Subject: [PATCH 271/367] Added a comment

---
 .../comment_3_886756620cdbb6ab838269fe2f00db4e._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_3_886756620cdbb6ab838269fe2f00db4e._comment

diff --git a/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_3_886756620cdbb6ab838269fe2f00db4e._comment b/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_3_886756620cdbb6ab838269fe2f00db4e._comment
new file mode 100644
index 0000000000..b5316c0de6
--- /dev/null
+++ b/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_3_886756620cdbb6ab838269fe2f00db4e._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="0xloem@0bd8a79a57e4f0dcade8fc81d162c37eae4d6730"
+ nickname="0xloem"
+ avatar="http://cdn.libravatar.org/avatar/b8c087f7c5e6a9358748f0727c077f3b"
+ subject="comment 3"
+ date="2016-12-11T00:26:41Z"
+ content="""
+Thank you so much for the prompt response.  My system wouldn't shut down cleanly after this, either, so there may have been something else screwy going on.  Still, I'll be using the build for ancient kernels exclusively for the near future.  Thank you.
+"""]]

From f58b13435ebe20c518031c99a0d019cc2ab32958 Mon Sep 17 00:00:00 2001
From: "0xloem@0bd8a79a57e4f0dcade8fc81d162c37eae4d6730" <0xloem@web>
Date: Sun, 11 Dec 2016 00:59:50 +0000
Subject: [PATCH 272/367] Added a comment: Verification

---
 .../comment_4_2440227de9b6bc77ae1c73b69a36f7a5._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_4_2440227de9b6bc77ae1c73b69a36f7a5._comment

diff --git a/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_4_2440227de9b6bc77ae1c73b69a36f7a5._comment b/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_4_2440227de9b6bc77ae1c73b69a36f7a5._comment
new file mode 100644
index 0000000000..801f702235
--- /dev/null
+++ b/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_4_2440227de9b6bc77ae1c73b69a36f7a5._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="0xloem@0bd8a79a57e4f0dcade8fc81d162c37eae4d6730"
+ nickname="0xloem"
+ avatar="http://cdn.libravatar.org/avatar/b8c087f7c5e6a9358748f0727c077f3b"
+ subject="Verification"
+ date="2016-12-11T00:59:50Z"
+ content="""
+I saw the new comment on the download page and tried running `git annex test`.  I can confirm that `git annex test` eventually segfaults using the normal build on my system, whereas it passes successfully using the 'ancient kernels' build.  The version strings output for the two are identical.
+"""]]

From 8283b1c924a6ea3b97b7e666d9a5c0c262019896 Mon Sep 17 00:00:00 2001
From: "0xloem@0bd8a79a57e4f0dcade8fc81d162c37eae4d6730" <0xloem@web>
Date: Sun, 11 Dec 2016 13:26:29 +0000
Subject: [PATCH 273/367] Added a comment: Nope a Fluke

---
 .../comment_5_c5e3dc25acf0cfb98d7068fe7f83e63a._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_5_c5e3dc25acf0cfb98d7068fe7f83e63a._comment

diff --git a/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_5_c5e3dc25acf0cfb98d7068fe7f83e63a._comment b/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_5_c5e3dc25acf0cfb98d7068fe7f83e63a._comment
new file mode 100644
index 0000000000..9dec23b00d
--- /dev/null
+++ b/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_5_c5e3dc25acf0cfb98d7068fe7f83e63a._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="0xloem@0bd8a79a57e4f0dcade8fc81d162c37eae4d6730"
+ nickname="0xloem"
+ avatar="http://cdn.libravatar.org/avatar/b8c087f7c5e6a9358748f0727c077f3b"
+ subject="Nope a Fluke"
+ date="2016-12-11T13:26:29Z"
+ content="""
+Apologies.  I can't reproduce the segfault running the tests again.  The corruption and crashing seems to have been some horrifying fluke.
+"""]]

From c3ab3c6682d2df96fad8ff476fa89b3b5777c626 Mon Sep 17 00:00:00 2001
From: "grawity@2ea26be48562f66fcb9b66307da72b1e2e37453f" <grawity@web>
Date: Sun, 11 Dec 2016 17:46:03 +0000
Subject: [PATCH 274/367]

---
 ...__breaks_user-specified_IgnoreUnknown.mdwn | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 doc/bugs/ssh___34__Include__34___breaks_user-specified_IgnoreUnknown.mdwn

diff --git a/doc/bugs/ssh___34__Include__34___breaks_user-specified_IgnoreUnknown.mdwn b/doc/bugs/ssh___34__Include__34___breaks_user-specified_IgnoreUnknown.mdwn
new file mode 100644
index 0000000000..b910ac4087
--- /dev/null
+++ b/doc/bugs/ssh___34__Include__34___breaks_user-specified_IgnoreUnknown.mdwn
@@ -0,0 +1,26 @@
+### Please describe the problem.
+
+The OpenSSH client parses configuration in a "first match wins" manner, and this also applies to `IgnoreUnknown`. This means that when git-annex's `Include ~/.ssh/config` is processed, any user-specified `IgnoreUnknown` setting in the global configuration will be ignored because it has already been set. As a result, every time git-annex runs ssh, it immediately exits with an error:
+
+[[!format text """
+drop vol3 somefile.mkv (locking vol5...) (lockcontent failed) (checking vol5...)
+/home/grawity/.ssh/config: line 217: Bad configuration option: gssapikeyexchange
+/home/grawity/.ssh/config: terminating, 1 bad configuration options
+failed
+"""]]
+
+To be fair, this might be an OpenSSH bug (IgnoreUnknown ought to be merged), but it seems git-annex is triggering it unnecessarily.
+
+### What steps will reproduce the problem?
+
+1. In `~/.ssh/config`, have some unrecognized options (e.g. `GSSAPIKeyExchange`) and a corresponding `IgnoreUnknown`.
+
+2. Try to use a git-annex feature which directly invokes ssh, e.g. get or drop.
+
+### What version of git-annex are you using? On what operating system?
+
+6.20161210 on Arch, but I think this was introduced in a 201611* release.
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Yes, it's been taking care of my archives for nearly a year.

From bb66e098b1a0a5ad21a34ca76789f9a77b36b913 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sun, 11 Dec 2016 15:37:52 -0400
Subject: [PATCH 275/367] linux standalone builds should have "unable to
 decommit memory" bug fixed

---
 CHANGELOG                                                 | 3 +++
 .../comment_3_a4499b5506c0624f01d436e14ccce909._comment   | 8 ++++++++
 doc/install/Linux_standalone.mdwn                         | 7 -------
 doc/news/version_6.20161210.mdwn                          | 5 ++++-
 4 files changed, 15 insertions(+), 8 deletions(-)
 create mode 100644 doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_3_a4499b5506c0624f01d436e14ccce909._comment

diff --git a/CHANGELOG b/CHANGELOG
index bb54989e86..21cd780a20 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,5 +1,8 @@
 git-annex (6.20161210) unstable; urgency=medium
 
+  * Linux standalone: Updated ghc to fix its "unable to decommit memory"
+    bug, which may have resulted in data loss when these builds were used
+    with Linux kernels older than 4.5.
   * enable-tor: New command, enables tor hidden service for P2P syncing.
   * p2p: New command, allows linking repositories using a P2P network.
   * remotedaemon: Serve tor hidden service.
diff --git a/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_3_a4499b5506c0624f01d436e14ccce909._comment b/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_3_a4499b5506c0624f01d436e14ccce909._comment
new file mode 100644
index 0000000000..7aea3d6af5
--- /dev/null
+++ b/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_3_a4499b5506c0624f01d436e14ccce909._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-12-11T19:35:21Z"
+ content="""
+All Linux standalone builds have been updated with a version of ghc that
+has that bug fixed. Can you please upgrade and verify it's fixed?
+"""]]
diff --git a/doc/install/Linux_standalone.mdwn b/doc/install/Linux_standalone.mdwn
index 5bad5f930b..223c8189b3 100644
--- a/doc/install/Linux_standalone.mdwn
+++ b/doc/install/Linux_standalone.mdwn
@@ -5,13 +5,6 @@ prebuilt tarball of the most recent release.
 This tarball should work on most Linux systems. It has basically no
 dependencies and is self-contained.
 
-**Warning** Linux kernels older than 4.5 may not run this build correctly, and
-<a href="http://git-annex.branchable.com/bugs/unable_to_decommit_memory__58___Invalid_argument/">data loss has been reported</a>
-using this build with the old kernel. We are working to resolve this
-problem, and if you use such a kernel, we recommend running
-`git annex test` to check git-annex works correctly, and/or 
-using the build for ancient kernels.
-
 * x86-32: [download tarball](https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-i386.tar.gz)
 * x86-64: [download tarball](https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-amd64.tar.gz)
 * x86-32, for ancient kernels: [download tarball](https://downloads.kitenet.net/git-annex/linux/current/git-annex-standalone-i386-ancient.tar.gz)
diff --git a/doc/news/version_6.20161210.mdwn b/doc/news/version_6.20161210.mdwn
index 5c3584d303..345d4fe4ce 100644
--- a/doc/news/version_6.20161210.mdwn
+++ b/doc/news/version_6.20161210.mdwn
@@ -1,5 +1,8 @@
 git-annex 6.20161210 released with [[!toggle text="these changes"]]
 [[!toggleable text="""
+   * Linux standalone: Updated ghc to fix its "unable to decommit memory"
+     bug, which may have resulted in data loss when these builds were used
+     with Linux kernels older than 4.5.
    * enable-tor: New command, enables tor hidden service for P2P syncing.
    * p2p: New command, allows linking repositories using a P2P network.
    * remotedaemon: Serve tor hidden service.
@@ -25,4 +28,4 @@ git-annex 6.20161210 released with [[!toggle text="these changes"]]
      (This was already done in v6 mode and direct mode.)
    * git-annex-shell, remotedaemon, git remote: Fix some memory DOS attacks.
    * Fix build with http-client 0.5.
-     Thanks, Alper Nebi Yasak."""]]
\ No newline at end of file
+     Thanks, Alper Nebi Yasak."""]]

From a52c0115811f7fa579572235690ed213a5fe2417 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sun, 11 Dec 2016 21:30:07 -0400
Subject: [PATCH 276/367] Debian: Build webapp on armel.

---
 CHANGELOG      |  6 ++++++
 debian/control | 22 +++++++++++-----------
 2 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index 21cd780a20..6978142805 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,9 @@
+git-annex (6.20161211) UNRELEASED; urgency=medium
+
+  * Debian: Build webapp on armel.
+
+ -- Joey Hess <id@joeyh.name>  Sun, 11 Dec 2016 21:29:51 -0400
+
 git-annex (6.20161210) unstable; urgency=medium
 
   * Linux standalone: Updated ghc to fix its "unable to decommit memory"
diff --git a/debian/control b/debian/control
index 58af2306cc..8be9fec995 100644
--- a/debian/control
+++ b/debian/control
@@ -32,17 +32,17 @@ Build-Depends:
 	libghc-stm-dev (>= 2.3),
 	libghc-dbus-dev (>= 0.10.7) [linux-any],
 	libghc-fdo-notify-dev (>= 0.3) [linux-any],
-	libghc-yesod-dev (>= 1.2.6.1)       [i386 amd64 arm64 armhf kfreebsd-amd64 kfreebsd-i386 mips mips64el mipsel powerpc ppc64el s390x],
-	libghc-yesod-core-dev (>= 1.2.19)   [i386 amd64 arm64 armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x],
-	libghc-yesod-form-dev (>= 1.3.15)   [i386 amd64 arm64 armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x],
-	libghc-yesod-static-dev (>= 1.2.4)  [i386 amd64 arm64 armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x],
-	libghc-yesod-default-dev (>= 1.2.0) [i386 amd64 arm64 armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x],
-	libghc-shakespeare-dev (>= 2.0.0)   [i386 amd64 arm64 armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x],
-	libghc-clientsession-dev            [i386 amd64 arm64 armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x],
-	libghc-warp-dev (>= 3.0.0.5)        [i386 amd64 arm64 armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x],
-	libghc-warp-tls-dev                 [i386 amd64 arm64 armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x],
-	libghc-wai-dev                      [i386 amd64 arm64 armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x],
-	libghc-wai-extra-dev                [i386 amd64 arm64 armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x],
+	libghc-yesod-dev (>= 1.2.6.1)       [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x],
+	libghc-yesod-core-dev (>= 1.2.19)   [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x],
+	libghc-yesod-form-dev (>= 1.3.15)   [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x],
+	libghc-yesod-static-dev (>= 1.2.4)  [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x],
+	libghc-yesod-default-dev (>= 1.2.0) [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x],
+	libghc-shakespeare-dev (>= 2.0.0)   [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x],
+	libghc-clientsession-dev            [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x],
+	libghc-warp-dev (>= 3.0.0.5)        [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x],
+	libghc-warp-tls-dev                 [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x],
+	libghc-wai-dev                      [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x],
+	libghc-wai-extra-dev                [i386 amd64 arm64 armel armhf kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x],
 	libghc-dav-dev (>= 1.0),
 	libghc-persistent-dev,
 	libghc-persistent-template-dev,

From ce2828a12f289a97ce3b6b99adaa2d44ecaea984 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 12 Dec 2016 13:27:39 -0400
Subject: [PATCH 277/367] finish fixing android build after tor merge

---
 Build/EvilSplicer.hs | 23 ++++++++---------------
 1 file changed, 8 insertions(+), 15 deletions(-)

diff --git a/Build/EvilSplicer.hs b/Build/EvilSplicer.hs
index 9a9dffa420..ca690c2501 100644
--- a/Build/EvilSplicer.hs
+++ b/Build/EvilSplicer.hs
@@ -486,18 +486,8 @@ mangleCode = flip_colon
 	 - So, we can put the semicolon at the start of every line
 	 - containing " -> " unless there's a "\ " first, or it's
 	 - all whitespace up until it.
-	 -
-	 - Except.. type signatures also contain " -> " sometimes starting
-	 - a line:
-	 -
-	 - forall foo =>
-	 -   Foo ->
-	 -
-	 - To avoid breaking these, look for the => on the previous line.
 	 -}
-	case_layout = parsecAndReplace $ do
-		void newline
-		lastline <- restOfLine
+	case_layout = skipfree $ parsecAndReplace $ do
 		void newline
 		indent1 <- many1 $ char ' '
 		prefix <- manyTill (noneOf "\n") (try (string "-> "))
@@ -507,9 +497,7 @@ mangleCode = flip_colon
 				then unexpected "lambda expression"
 				else if null prefix
 					then unexpected "second line of lambda"
-					else if "=>" `isSuffixOf` lastline
-						then unexpected "probably type signature"
-						else return $ "\n" ++ lastline ++ "\n" ++ indent1 ++ "; " ++ prefix ++ " -> "
+					else return $ "\n" ++ indent1 ++ "; " ++ prefix ++ " -> "
 	{- Sometimes cases themselves span multiple lines:
 	 -
 	 - Nothing
@@ -520,7 +508,7 @@ mangleCode = flip_colon
 	 -        var var
 	 -   -> foo
 	 -}
-	case_layout_multiline = parsecAndReplace $ do
+	case_layout_multiline = skipfree $ parsecAndReplace $ do
 		void newline
 		indent1 <- many1 $ char ' '
 		firstline <- restofline
@@ -533,6 +521,11 @@ mangleCode = flip_colon
 			else return $ "\n" ++ indent1 ++ "; " ++ firstline ++ "\n"
 				++ indent1 ++ indent2 ++ "-> "
 
+	{- Type definitions for free monads triggers the case_* hacks, avoid. -}
+	skipfree f s
+		| "MonadFree" `isInfixOf` s = s
+		| otherwise = f s
+
 	{- (foo, \ -> bar) is not valid haskell, GHC.
 	 - Change to (foo, bar)
 	 -

From 63a49d4df8a1f20be38e9bba907ecad24a34915e Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 12 Dec 2016 13:31:58 -0400
Subject: [PATCH 278/367] comment

---
 ...omment_7_e31ee8f49bf5f73620209c524f1edb3d._comment | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_7_e31ee8f49bf5f73620209c524f1edb3d._comment

diff --git a/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_7_e31ee8f49bf5f73620209c524f1edb3d._comment b/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_7_e31ee8f49bf5f73620209c524f1edb3d._comment
new file mode 100644
index 0000000000..08e3364cac
--- /dev/null
+++ b/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_7_e31ee8f49bf5f73620209c524f1edb3d._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 7"""
+ date="2016-12-12T17:30:03Z"
+ content="""
+Can you please check if the current builds still have the "unable to
+decommit memory" problem or not?
+
+(What it does after that error is probably nondeterministic, fixing that
+error is the crucial thing.)
+"""]]

From 6f502c859b758e7cb6632b7546a22fdf17067ac4 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 12 Dec 2016 17:21:32 -0400
Subject: [PATCH 279/367] todo

---
 doc/todo/smudge.mdwn | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/todo/smudge.mdwn b/doc/todo/smudge.mdwn
index 78a20fd6d9..6117224903 100644
--- a/doc/todo/smudge.mdwn
+++ b/doc/todo/smudge.mdwn
@@ -37,6 +37,10 @@ git-annex should use smudge/clean filters.
   And developed a patch set:
   <http://thread.gmane.org/gmane.comp.version-control.git/297475>
 
+* Implement git's new `filter.<driver>.process` interface, which will
+  let only 1 git-annex process be started by git when processing
+  multiple files, and so should be faster.
+
 * Checking out a different branch causes git to smudge all changed files,
   and write their content. This does not honor annex.thin. A warning
   message is printed in this case.  

From bc3e1b89462b6deab5c602256844a28d2a67817b Mon Sep 17 00:00:00 2001
From: annexuser <annexuser@web>
Date: Tue, 13 Dec 2016 02:00:08 +0000
Subject: [PATCH 280/367] Added a comment

---
 .../comment_1_526f6a007f44f389ef7c904024752541._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/more_intelligent_copy_/comment_1_526f6a007f44f389ef7c904024752541._comment

diff --git a/doc/forum/more_intelligent_copy_/comment_1_526f6a007f44f389ef7c904024752541._comment b/doc/forum/more_intelligent_copy_/comment_1_526f6a007f44f389ef7c904024752541._comment
new file mode 100644
index 0000000000..9b5866c5c5
--- /dev/null
+++ b/doc/forum/more_intelligent_copy_/comment_1_526f6a007f44f389ef7c904024752541._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="annexuser"
+ avatar="http://cdn.libravatar.org/avatar/6ae692503ee113b1ab7b329b40084d5c"
+ subject="comment 1"
+ date="2016-12-13T02:00:08Z"
+ content="""
+git annex copy --to REMOTE FILES --fast
+"""]]

From 6ff7b0e81b2f09394037ce80213e367f929fb624 Mon Sep 17 00:00:00 2001
From: "https://singpolyma.net/" <https://singpolyma.net/@web>
Date: Tue, 13 Dec 2016 02:38:44 +0000
Subject: [PATCH 281/367]

---
 doc/forum/GIT__95__SSH.mdwn | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 doc/forum/GIT__95__SSH.mdwn

diff --git a/doc/forum/GIT__95__SSH.mdwn b/doc/forum/GIT__95__SSH.mdwn
new file mode 100644
index 0000000000..968385d084
--- /dev/null
+++ b/doc/forum/GIT__95__SSH.mdwn
@@ -0,0 +1,3 @@
+Is there any way to get git-annex to respect the GIT_SSH environment variable just like git?  This would be super helpful for specifying ssh options (keys in my application)
+
+Hacks like a ~/.ssh/config with a fake hostname to specify the key are not appropriate in my case, because keys are generated on the fly by a server-side application that may be distributed across multiple servers.  So I really need to be able to specify a key file at run time and then use the normal remote URI.  I can do this with git and GIT_SSH, but have not found a solution for git-annex yet.

From d35d87d8d868327f043fb3bc054a68b893364cb6 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 13 Dec 2016 10:40:50 -0400
Subject: [PATCH 282/367] analysis

---
 ...t_1_627bb742a5042741e9a1c294addd69b2._comment | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 doc/bugs/Metadata_values_get_stuck_when_repeatedly_modified_in_the_same_batch_mode_run/comment_1_627bb742a5042741e9a1c294addd69b2._comment

diff --git a/doc/bugs/Metadata_values_get_stuck_when_repeatedly_modified_in_the_same_batch_mode_run/comment_1_627bb742a5042741e9a1c294addd69b2._comment b/doc/bugs/Metadata_values_get_stuck_when_repeatedly_modified_in_the_same_batch_mode_run/comment_1_627bb742a5042741e9a1c294addd69b2._comment
new file mode 100644
index 0000000000..8f84474915
--- /dev/null
+++ b/doc/bugs/Metadata_values_get_stuck_when_repeatedly_modified_in_the_same_batch_mode_run/comment_1_627bb742a5042741e9a1c294addd69b2._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-12-13T14:40:02Z"
+ content="""
+I thought this would involve the journal, but it seems not; same
+behavior occurs if the journal is committed after each metadata change.
+
+Looking at the new metadata value in the case where a and c both get set,
+it is:
+
+	MetaData (fromList [(MetaField "f",fromList [MetaValue (CurrentlySet True) "a",MetaValue (CurrentlySet False) "c"])])
+
+That is supposed to unset c, with the CurrentlySet False, but instead c
+remains set somehow.
+"""]]

From e08761288d3a200a4983f9f6c3e5fdb2a8c7a39b Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 13 Dec 2016 10:49:39 -0400
Subject: [PATCH 283/367] analysis

---
 .../comment_1_627bb742a5042741e9a1c294addd69b2._comment     | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/doc/bugs/Metadata_values_get_stuck_when_repeatedly_modified_in_the_same_batch_mode_run/comment_1_627bb742a5042741e9a1c294addd69b2._comment b/doc/bugs/Metadata_values_get_stuck_when_repeatedly_modified_in_the_same_batch_mode_run/comment_1_627bb742a5042741e9a1c294addd69b2._comment
index 8f84474915..9541f1d744 100644
--- a/doc/bugs/Metadata_values_get_stuck_when_repeatedly_modified_in_the_same_batch_mode_run/comment_1_627bb742a5042741e9a1c294addd69b2._comment
+++ b/doc/bugs/Metadata_values_get_stuck_when_repeatedly_modified_in_the_same_batch_mode_run/comment_1_627bb742a5042741e9a1c294addd69b2._comment
@@ -13,4 +13,10 @@ it is:
 
 That is supposed to unset c, with the CurrentlySet False, but instead c
 remains set somehow.
+
+Aha, the use of `addMetaData'` causes the bug. That reuses the same
+timestamp, and indeed the same timestamp is used for all the batch
+changes. With the same timestamp for the log line that sets c as the line
+that removes it, it's indeterminite which line will be acted on first, and
+so the removal can be processed before the addition, leaving c "stuck".
 """]]

From f01b4cbf7c27162e08348b7a22c3eb841f3619b9 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 13 Dec 2016 10:57:11 -0400
Subject: [PATCH 284/367] analysis

---
 .../comment_1_627bb742a5042741e9a1c294addd69b2._comment         | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/Metadata_values_get_stuck_when_repeatedly_modified_in_the_same_batch_mode_run/comment_1_627bb742a5042741e9a1c294addd69b2._comment b/doc/bugs/Metadata_values_get_stuck_when_repeatedly_modified_in_the_same_batch_mode_run/comment_1_627bb742a5042741e9a1c294addd69b2._comment
index 9541f1d744..4f82db153c 100644
--- a/doc/bugs/Metadata_values_get_stuck_when_repeatedly_modified_in_the_same_batch_mode_run/comment_1_627bb742a5042741e9a1c294addd69b2._comment
+++ b/doc/bugs/Metadata_values_get_stuck_when_repeatedly_modified_in_the_same_batch_mode_run/comment_1_627bb742a5042741e9a1c294addd69b2._comment
@@ -19,4 +19,6 @@ timestamp, and indeed the same timestamp is used for all the batch
 changes. With the same timestamp for the log line that sets c as the line
 that removes it, it's indeterminite which line will be acted on first, and
 so the removal can be processed before the addition, leaving c "stuck".
+
+Fixing..
 """]]

From d9490685fd5311c59ad4194ea162a42a0ca6e8b8 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 13 Dec 2016 11:07:49 -0400
Subject: [PATCH 285/367] metadata --batch: Fix bug when conflicting metadata
 changes were made in the same batch run.

1 microsecond delay is ugly.. but, maintaining an queue of a list of timestamps
and taking a new one from the queue each time around, or maintaining a timestamp
counter, would probably be slower.
---
 CHANGELOG                                     |  2 +
 Command/MetaData.hs                           | 47 +++++++++++--------
 ...y_modified_in_the_same_batch_mode_run.mdwn |  1 +
 3 files changed, 31 insertions(+), 19 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index 6978142805..839af6559a 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,8 @@
 git-annex (6.20161211) UNRELEASED; urgency=medium
 
   * Debian: Build webapp on armel.
+  * metadata --batch: Fix bug when conflicting metadata changes were
+    made in the same batch run.
 
  -- Joey Hess <id@joeyh.name>  Sun, 11 Dec 2016 21:29:51 -0400
 
diff --git a/Command/MetaData.hs b/Command/MetaData.hs
index 04d859e4cf..ebb9d0f174 100644
--- a/Command/MetaData.hs
+++ b/Command/MetaData.hs
@@ -20,6 +20,7 @@ import qualified Data.Text as T
 import qualified Data.ByteString.Lazy.UTF8 as BU
 import Data.Time.Clock.POSIX
 import Data.Aeson
+import Control.Concurrent
 
 cmd :: Command
 cmd = withGlobalOptions ([jsonOption] ++ annexedMatchingOptions) $ 
@@ -65,23 +66,22 @@ optParser desc = MetaDataOptions
 			)
 
 seek :: MetaDataOptions -> CommandSeek
-seek o = do
-	now <- liftIO getPOSIXTime
-	case batchOption o of
-		NoBatch -> do
-			let seeker = case getSet o of
-				Get _ -> withFilesInGit
-				GetAll -> withFilesInGit
-				Set _ -> withFilesInGitNonRecursive
-					"Not recursively setting metadata. Use --force to do that."
-			withKeyOptions (keyOptions o) False
-				(startKeys now o)
-				(seeker $ whenAnnexed $ start now o)
-				(forFiles o)
-		Batch -> withMessageState $ \s -> case outputType s of
-			JSONOutput _ -> batchInput parseJSONInput $
-				commandAction . startBatch now
-			_ -> giveup "--batch is currently only supported in --json mode"
+seek o = case batchOption o of
+	NoBatch -> do
+		now <- liftIO getPOSIXTime
+		let seeker = case getSet o of
+			Get _ -> withFilesInGit
+			GetAll -> withFilesInGit
+			Set _ -> withFilesInGitNonRecursive
+				"Not recursively setting metadata. Use --force to do that."
+		withKeyOptions (keyOptions o) False
+			(startKeys now o)
+			(seeker $ whenAnnexed $ start now o)
+			(forFiles o)
+	Batch -> withMessageState $ \s -> case outputType s of
+		JSONOutput _ -> batchInput parseJSONInput $
+			commandAction . startBatch
+		_ -> giveup "--batch is currently only supported in --json mode"
 
 start :: POSIXTime -> MetaDataOptions -> FilePath -> Key -> CommandStart
 start now o file k = startKeys now o k (mkActionItem afile)
@@ -150,8 +150,8 @@ parseJSONInput i = do
 		(Nothing, Just f) -> Right (Left f, m)
 		(Nothing, Nothing) -> Left "JSON input is missing either file or key"
 
-startBatch :: POSIXTime -> (Either FilePath Key, MetaData) -> CommandStart
-startBatch now (i, (MetaData m)) = case i of
+startBatch :: (Either FilePath Key, MetaData) -> CommandStart
+startBatch (i, (MetaData m)) = case i of
 	Left f -> do
 		mk <- lookupFile f
 		case mk of
@@ -169,6 +169,15 @@ startBatch now (i, (MetaData m)) = case i of
 			, keyOptions = Nothing
 			, batchOption = NoBatch
 			}
+		now <- liftIO getPOSIXTime
+		-- It would be bad if two batch mode changes used exactly
+		-- the same timestamp, since the order of adds and removals
+		-- of the same metadata value would then be indeterminate.
+		-- To guarantee that never happens, delay 1 microsecond,
+		-- so the timestamp will always be different. This is
+		-- probably less expensive than cleaner methods,
+		-- such as taking from a list of increasing timestamps.
+		liftIO $ threadDelay 1
 		next $ perform now o k
 	mkModMeta (f, s)
 		| S.null s = DelMeta f Nothing
diff --git a/doc/bugs/Metadata_values_get_stuck_when_repeatedly_modified_in_the_same_batch_mode_run.mdwn b/doc/bugs/Metadata_values_get_stuck_when_repeatedly_modified_in_the_same_batch_mode_run.mdwn
index 1ec07dccbe..26790bea3f 100644
--- a/doc/bugs/Metadata_values_get_stuck_when_repeatedly_modified_in_the_same_batch_mode_run.mdwn
+++ b/doc/bugs/Metadata_values_get_stuck_when_repeatedly_modified_in_the_same_batch_mode_run.mdwn
@@ -82,3 +82,4 @@ I love the metadata functionality so much that I wrote [[tips/a_gui_for_metadata
 Metadata driven views are awesome (but I don't like the entire folder hierarchy being appended to the filename).
 I haven't used the other commands much since I have not yet organized most of my stuff (and their naively copy-pasted backups), but I am glad I discovered git-annex before I began organizing.
 
+> [[fixed|done]] --[[Joey]]

From b55d5a216d9179bec8f18fc32e4e1552d9d5541d Mon Sep 17 00:00:00 2001
From: "0xloem@0bd8a79a57e4f0dcade8fc81d162c37eae4d6730" <0xloem@web>
Date: Tue, 13 Dec 2016 15:52:35 +0000
Subject: [PATCH 286/367] Added a comment

---
 .../comment_8_038a8e39ec0e91cb04af738eaf9095e1._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_8_038a8e39ec0e91cb04af738eaf9095e1._comment

diff --git a/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_8_038a8e39ec0e91cb04af738eaf9095e1._comment b/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_8_038a8e39ec0e91cb04af738eaf9095e1._comment
new file mode 100644
index 0000000000..6d8994155f
--- /dev/null
+++ b/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_8_038a8e39ec0e91cb04af738eaf9095e1._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="0xloem@0bd8a79a57e4f0dcade8fc81d162c37eae4d6730"
+ nickname="0xloem"
+ avatar="http://cdn.libravatar.org/avatar/b8c087f7c5e6a9358748f0727c077f3b"
+ subject="comment 8"
+ date="2016-12-13T15:52:34Z"
+ content="""
+It looks like the errors are gone.  Thank you so much.
+"""]]

From 59fead6da3a6edcd2add70c48e68ddf8025d41cb Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 13 Dec 2016 11:56:23 -0400
Subject: [PATCH 287/367] Pass annex.web-options to wget and curl after other
 options, so that eg --no-show-progress can be set by the user to disable the
 default --show-progress.

---
 CHANGELOG                                     |  3 +++
 Utility/Url.hs                                |  2 +-
 doc/bugs/wget_always_shows_progress_bar.mdwn  |  1 +
 ..._d40883c9f9aade47112a0479ad56ed06._comment | 19 +++++++++++++++++++
 4 files changed, 24 insertions(+), 1 deletion(-)
 create mode 100644 doc/bugs/wget_always_shows_progress_bar/comment_1_d40883c9f9aade47112a0479ad56ed06._comment

diff --git a/CHANGELOG b/CHANGELOG
index 839af6559a..bd31348508 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -3,6 +3,9 @@ git-annex (6.20161211) UNRELEASED; urgency=medium
   * Debian: Build webapp on armel.
   * metadata --batch: Fix bug when conflicting metadata changes were
     made in the same batch run.
+  * Pass annex.web-options to wget and curl after other options, so that
+    eg --no-show-progress can be set by the user to disable the default
+    --show-progress.
 
  -- Joey Hess <id@joeyh.name>  Sun, 11 Dec 2016 21:29:51 -0400
 
diff --git a/Utility/Url.hs b/Utility/Url.hs
index d0e1b3739c..a4523d73f7 100644
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -303,7 +303,7 @@ download' quiet url file uo = do
 	 - it was asked to write to a file elsewhere. -}
 	go cmd opts = withTmpDir "downloadurl" $ \tmp -> do
 		absfile <- absPath file
-		let ps = addUserAgent uo $ reqParams uo++opts++[File absfile, File url]
+		let ps = addUserAgent uo $ opts++reqParams uo++[File absfile, File url]
 		boolSystem' cmd ps $ \p -> p { cwd = Just tmp }
 	
 	quietopt s
diff --git a/doc/bugs/wget_always_shows_progress_bar.mdwn b/doc/bugs/wget_always_shows_progress_bar.mdwn
index 2829b39b0e..708db8906f 100644
--- a/doc/bugs/wget_always_shows_progress_bar.mdwn
+++ b/doc/bugs/wget_always_shows_progress_bar.mdwn
@@ -22,3 +22,4 @@ parallel git annex importfeed --relaxed --quiet ::: http://feeds.feedburner.com/
 I love git annex and use it daily.
 
 
+> [[done]] --[[Joey]]
diff --git a/doc/bugs/wget_always_shows_progress_bar/comment_1_d40883c9f9aade47112a0479ad56ed06._comment b/doc/bugs/wget_always_shows_progress_bar/comment_1_d40883c9f9aade47112a0479ad56ed06._comment
new file mode 100644
index 0000000000..c2e6eb53f9
--- /dev/null
+++ b/doc/bugs/wget_always_shows_progress_bar/comment_1_d40883c9f9aade47112a0479ad56ed06._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-12-13T15:49:20Z"
+ content="""
+Since 2014, git-annex has used wget -q --show-progress to get a
+progress bar without the several other lines of output it would normally
+display. Whether a given git-annex build does this depends on what
+version of wget it saw at configure time.
+
+Running git-annex with --quiet will disable the wget progress bar (and
+other git-annex output). This seems like the thing to do if you're running
+git-annex concurrently. (Of course, git-annex also has its own built-in
+concurrency with -J which can display multiple download progress bars in a
+nice way.)
+
+Still, might as well make the web-options come after the default options so
+they can be overridden. Doing so.
+"""]]

From d87abe61f69858161361affb55a89418b834a4e1 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 13 Dec 2016 11:57:32 -0400
Subject: [PATCH 288/367] comment

---
 .../comment_1_f155ffc7dbd074964dd53165274ec8a0._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/todo/Long_Running_Filter_Process/comment_1_f155ffc7dbd074964dd53165274ec8a0._comment

diff --git a/doc/todo/Long_Running_Filter_Process/comment_1_f155ffc7dbd074964dd53165274ec8a0._comment b/doc/todo/Long_Running_Filter_Process/comment_1_f155ffc7dbd074964dd53165274ec8a0._comment
new file mode 100644
index 0000000000..34d05d7711
--- /dev/null
+++ b/doc/todo/Long_Running_Filter_Process/comment_1_f155ffc7dbd074964dd53165274ec8a0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-12-13T15:57:05Z"
+ content="""
+Yes, this will make [[smudge]] faster when eg checking out a lot of working
+tree changes. I will need to add support for it.
+"""]]

From 11fd49870c7eb19ec5fff4751d5f02e260c965bf Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 13 Dec 2016 11:58:56 -0400
Subject: [PATCH 289/367] comment

---
 .../comment_2_32c45cc852a17e837f72dd8769a25781._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/bugs/cabal_constraints_for_aws_and_esqueleto/comment_2_32c45cc852a17e837f72dd8769a25781._comment

diff --git a/doc/bugs/cabal_constraints_for_aws_and_esqueleto/comment_2_32c45cc852a17e837f72dd8769a25781._comment b/doc/bugs/cabal_constraints_for_aws_and_esqueleto/comment_2_32c45cc852a17e837f72dd8769a25781._comment
new file mode 100644
index 0000000000..ada283d8b3
--- /dev/null
+++ b/doc/bugs/cabal_constraints_for_aws_and_esqueleto/comment_2_32c45cc852a17e837f72dd8769a25781._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-12-13T15:58:25Z"
+ content="""
+This got fixed in the meantime. Note that posting comments to a bug that
+has already been closed is a good way to get new problems not to be
+noticed..
+"""]]

From 1d516069d812bb25e8559d916e0d3c3e229a9d0c Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 13 Dec 2016 12:00:37 -0400
Subject: [PATCH 290/367] close

---
 doc/bugs/addurl_pathdepth_description_misleading.mdwn | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/bugs/addurl_pathdepth_description_misleading.mdwn b/doc/bugs/addurl_pathdepth_description_misleading.mdwn
index 531e43cbf0..7bb9d582a1 100644
--- a/doc/bugs/addurl_pathdepth_description_misleading.mdwn
+++ b/doc/bugs/addurl_pathdepth_description_misleading.mdwn
@@ -11,3 +11,7 @@ This isn't how it behaves. It would be more accurate as (emphasis on changes):
 > For example, adding the url http://www.example.com/dir/subdir/bigfile with --pathdepth=1 will use "**dir_subdir_bigfile**", while --pathdepth=3 will use "bigfile".
 
 For what I am doing (adding a directory tree with addurl and file:// URLs), I'd actually like the behaviour described (to recreate the tree), but I'm not sure which one was the *intended* behaviour..
+
+> [[done]]; bug report didn't show what was wrong; I can see nothing wrong;
+> bug reporter cannot seem to remember what was wrong. Probably user error.
+> --[[Joey]]

From 48d9624a2d77c9ffef69233289c4f9f93bf8733b Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 13 Dec 2016 12:05:05 -0400
Subject: [PATCH 291/367] Revert ServerAliveInterval

Revert ServerAliveInterval change in 6.20161111, which caused problems
with too many old versions of ssh and unusual ssh configurations.

It should have not been needed anyway since ssh is supposted to
have TCPKeepAlive enabled by default.
---
 Annex/Locations.hs                            |  5 ---
 Annex/Ssh.hs                                  | 39 ++++---------------
 CHANGELOG                                     |  4 ++
 .../Allow_automatic_retry_git_annex_get.mdwn  |  2 -
 ..._899b66a20b8e29a23068d249a461c19f._comment | 16 ++++++++
 ..._failed_due_to_unsupported_ssh_option.mdwn |  2 +
 ...__breaks_user-specified_IgnoreUnknown.mdwn |  4 ++
 ...__Include__34___feature_broke_Android.mdwn |  2 +
 8 files changed, 35 insertions(+), 39 deletions(-)
 create mode 100644 doc/bugs/Allow_automatic_retry_git_annex_get/comment_4_899b66a20b8e29a23068d249a461c19f._comment

diff --git a/Annex/Locations.hs b/Annex/Locations.hs
index 9f829fda19..a6af4d417f 100644
--- a/Annex/Locations.hs
+++ b/Annex/Locations.hs
@@ -63,7 +63,6 @@ module Annex.Locations (
 	gitAnnexUrlFile,
 	gitAnnexTmpCfgFile,
 	gitAnnexSshDir,
-	gitAnnexSshConfig,
 	gitAnnexRemotesDir,
 	gitAnnexAssistantDefaultDir,
 	HashLevels(..),
@@ -403,10 +402,6 @@ gitAnnexTmpCfgFile r = gitAnnexDir r </> "config.tmp"
 gitAnnexSshDir :: Git.Repo -> FilePath
 gitAnnexSshDir r = addTrailingPathSeparator $ gitAnnexDir r </> "ssh"
 
-{- .git/annex/ssh.config is used to configure ssh. -}
-gitAnnexSshConfig :: Git.Repo -> FilePath
-gitAnnexSshConfig r = gitAnnexDir r </> "ssh.config"
-
 {- .git/annex/remotes/ is used for remote-specific state. -}
 gitAnnexRemotesDir :: Git.Repo -> FilePath
 gitAnnexRemotesDir r = addTrailingPathSeparator $ gitAnnexDir r </> "remotes"
diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs
index 4377de4c58..4f879436b5 100644
--- a/Annex/Ssh.hs
+++ b/Annex/Ssh.hs
@@ -33,7 +33,6 @@ import qualified Git.Url
 import Config
 import Annex.Path
 import Utility.Env
-import Utility.Tmp
 import Types.CleanupActions
 import Git.Env
 #ifndef mingw32_HOST_OS
@@ -50,37 +49,13 @@ sshOptions (host, port) gc opts = go =<< sshCachingInfo (host, port)
 	go (Just socketfile, params) = do
 		prepSocket socketfile
 		ret params
-	ret ps = do
-		overideconfigfile <- fromRepo gitAnnexSshConfig
-		-- We assume that the file content does not change.
-		-- If it did, a more expensive test would be needed.
-		liftIO $ unlessM (doesFileExist overideconfigfile) $
-			viaTmp writeFile overideconfigfile $ unlines
-				-- Make old version of ssh that does
-				-- not know about Include ignore those
-				-- entries.
-				[ "IgnoreUnknown Include"
-				-- ssh expands "~"
-				, "Include ~/.ssh/config"
-				-- ssh will silently skip the file
-				-- if it does not exist
-				, "Include /etc/ssh/ssh_config"
-				-- Everything below this point is only
-				-- used if there's no setting for it in
-				-- the above files.
-				--
-				-- Make sure that ssh detects stalled
-				-- connections.
-				, "ServerAliveInterval 60"
-				]
-		return $ concat
-			[ ps
-			, [Param "-F", File overideconfigfile]
-			, map Param (remoteAnnexSshOptions gc)
-			, opts
-			, portParams port
-			, [Param "-T"]
-			]
+	ret ps = return $ concat
+		[ ps
+		, map Param (remoteAnnexSshOptions gc)
+		, opts
+		, portParams port
+		, [Param "-T"]
+		]
 
 {- Returns a filename to use for a ssh connection caching socket, and
  - parameters to enable ssh connection caching. -}
diff --git a/CHANGELOG b/CHANGELOG
index bd31348508..4bd69705b0 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -6,6 +6,10 @@ git-annex (6.20161211) UNRELEASED; urgency=medium
   * Pass annex.web-options to wget and curl after other options, so that
     eg --no-show-progress can be set by the user to disable the default
     --show-progress.
+  * Revert ServerAliveInterval change in 6.20161111, which caused problems
+    with too many old versions of ssh and unusual ssh configurations.
+    It should have not been needed anyway since ssh is supposted to
+    have TCPKeepAlive enabled by default.
 
  -- Joey Hess <id@joeyh.name>  Sun, 11 Dec 2016 21:29:51 -0400
 
diff --git a/doc/bugs/Allow_automatic_retry_git_annex_get.mdwn b/doc/bugs/Allow_automatic_retry_git_annex_get.mdwn
index 0e85b7acfb..c6e406b490 100644
--- a/doc/bugs/Allow_automatic_retry_git_annex_get.mdwn
+++ b/doc/bugs/Allow_automatic_retry_git_annex_get.mdwn
@@ -59,5 +59,3 @@ SHA256E-s41311329--69c3b054a3fe9676133605730d85b7fcef8696f6782d402a524e41b836253
 
 
 [[!meta title="Detect stalled transfer and retry or abort it"]]
-
-> [[done]] --[[Joey]]
diff --git a/doc/bugs/Allow_automatic_retry_git_annex_get/comment_4_899b66a20b8e29a23068d249a461c19f._comment b/doc/bugs/Allow_automatic_retry_git_annex_get/comment_4_899b66a20b8e29a23068d249a461c19f._comment
new file mode 100644
index 0000000000..6d25069233
--- /dev/null
+++ b/doc/bugs/Allow_automatic_retry_git_annex_get/comment_4_899b66a20b8e29a23068d249a461c19f._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2016-12-13T16:05:42Z"
+ content="""
+Could the original bug reporter please show what your ~/.ssh/config
+contains? As far as I can tell, ssh's TCPKeepAlive option, which is
+supposed to be enabled by default, unless you have disabled it, should
+avoid such problems.
+
+It may also help to set ServerAliveInterval.
+
+Unfortunately, my attempt to make git-annex set ServerAliveInterval
+when running ssh broke too many systems with old sshed, and I have had to
+revert it.
+"""]]
diff --git a/doc/bugs/git_annex_init_failed_due_to_unsupported_ssh_option.mdwn b/doc/bugs/git_annex_init_failed_due_to_unsupported_ssh_option.mdwn
index eb961030cd..f4339bff79 100644
--- a/doc/bugs/git_annex_init_failed_due_to_unsupported_ssh_option.mdwn
+++ b/doc/bugs/git_annex_init_failed_due_to_unsupported_ssh_option.mdwn
@@ -65,3 +65,5 @@ I would just like to be sure nothing else is hidden.
 > .git/config to remove that in order to recover from the problem, so might
 > as well remove `.git/annex/ssh_config` too.
 > --[[Joey]]
+
+>> Fixed more by stopping using `.git/annex/ssh_config` at all.  --[[Joey]]
diff --git a/doc/bugs/ssh___34__Include__34___breaks_user-specified_IgnoreUnknown.mdwn b/doc/bugs/ssh___34__Include__34___breaks_user-specified_IgnoreUnknown.mdwn
index b910ac4087..20878cb21e 100644
--- a/doc/bugs/ssh___34__Include__34___breaks_user-specified_IgnoreUnknown.mdwn
+++ b/doc/bugs/ssh___34__Include__34___breaks_user-specified_IgnoreUnknown.mdwn
@@ -24,3 +24,7 @@ To be fair, this might be an OpenSSH bug (IgnoreUnknown ought to be merged), but
 ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
 
 Yes, it's been taking care of my archives for nearly a year.
+
+> How annoying, ssh seems to make it impossible to set this in a way that
+> doesn't break some configurations. Oh well, gave up on setting it
+> and removed the code to do so. [[done]] --[[Joey]]
diff --git a/doc/bugs/ssh___34__Include__34___feature_broke_Android.mdwn b/doc/bugs/ssh___34__Include__34___feature_broke_Android.mdwn
index 9be4d4318d..ca5b86c87d 100644
--- a/doc/bugs/ssh___34__Include__34___feature_broke_Android.mdwn
+++ b/doc/bugs/ssh___34__Include__34___feature_broke_Android.mdwn
@@ -6,3 +6,5 @@ Unfortunately, the Android apk (which I got from https://downloads.kitenet.net/g
 I had to edit .git/annex/ssh.config to comment out the three offending lines and then append the contents of ~/.ssh/config to get git-annex working again.
 
 (This is on a Nexus 10 running stock, though I doubt it matters)
+
+> Reverted use of this feature for now.[[done]] --[[Joey]]

From 61944ad12c34b52af36e5d8cffd1b27762107508 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 13 Dec 2016 12:19:44 -0400
Subject: [PATCH 292/367] close

---
 doc/bugs/unable_to_decommit_memory__58___Invalid_argument.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/bugs/unable_to_decommit_memory__58___Invalid_argument.mdwn b/doc/bugs/unable_to_decommit_memory__58___Invalid_argument.mdwn
index 572e0406a3..6cd90264cc 100644
--- a/doc/bugs/unable_to_decommit_memory__58___Invalid_argument.mdwn
+++ b/doc/bugs/unable_to_decommit_memory__58___Invalid_argument.mdwn
@@ -26,3 +26,6 @@ It looks like this is a problem that occurs only on kernels < 4.5, when ghc is b
 ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
 
 Git annex rocks!
+
+> [[fixed|done]]; fix is confirmed, all linux standalone builds are updated
+> (and I pinged Yoh to update the neurodebian standalone build). --[[Joey]]

From 3bf31075edf9dbca8f8cc09ce6a1317ea2dab016 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 13 Dec 2016 12:43:20 -0400
Subject: [PATCH 293/367] comment

---
 ...mment_1_bd56607f228f3480f1355e3bdb755410._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/bugs/Inconsistent_results_between_git-annex-fsck_and_git-annex-whereis/comment_1_bd56607f228f3480f1355e3bdb755410._comment

diff --git a/doc/bugs/Inconsistent_results_between_git-annex-fsck_and_git-annex-whereis/comment_1_bd56607f228f3480f1355e3bdb755410._comment b/doc/bugs/Inconsistent_results_between_git-annex-fsck_and_git-annex-whereis/comment_1_bd56607f228f3480f1355e3bdb755410._comment
new file mode 100644
index 0000000000..d65f39fd0a
--- /dev/null
+++ b/doc/bugs/Inconsistent_results_between_git-annex-fsck_and_git-annex-whereis/comment_1_bd56607f228f3480f1355e3bdb755410._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-12-13T16:42:08Z"
+ content="""
+The obvious reason for this would be if the file no longer points to that
+same key. Perhaps the file got modified and the key is the old version of
+the file.
+
+That would explain everything you showed, so currently I don't see any
+bug..
+"""]]

From 1d82cf39cacab8c799b4df3320013a549ce8fb81 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 13 Dec 2016 12:48:11 -0400
Subject: [PATCH 294/367] response

---
 ..._a4a0491a7dcee2e7b7786127518866af._comment | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 doc/special_remotes/rsync/comment_15_a4a0491a7dcee2e7b7786127518866af._comment

diff --git a/doc/special_remotes/rsync/comment_15_a4a0491a7dcee2e7b7786127518866af._comment b/doc/special_remotes/rsync/comment_15_a4a0491a7dcee2e7b7786127518866af._comment
new file mode 100644
index 0000000000..2f68c3f574
--- /dev/null
+++ b/doc/special_remotes/rsync/comment_15_a4a0491a7dcee2e7b7786127518866af._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 15"""
+ date="2016-12-13T16:43:42Z"
+ content="""
+@davidriod you can do things like this with special remotes, as long
+as the special remotes are not encrypted.
+
+I don't really recommend it. With such a shared special remote R and two
+disconnected git repos -- call them A and B, some confusing situations can
+occur. For example, the only copies of some files may be 
+on special remote R and git repo B. A knows about the copy in R, so 
+git-annex is satisfied there is one copy of the file. But now, B can drop
+the content from R, which is allowed as the content is in B. A is then left
+unable to recover the content of the files at all, since they have been
+removed from R.
+
+Better to connect the two repositories A and B, even if you do work in
+two separate branches. Then if a file ends up located only on B, A will be
+able to say where it is, and could even get it from B (if B was set up as a
+remote).
+"""]]

From 6584d875d639663d433a1031d066d64880918248 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 13 Dec 2016 12:53:12 -0400
Subject: [PATCH 295/367] response

---
 ...nt_2_7b3f5d2e9de4b13de821177db2f57bcd._comment | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/forum/more_intelligent_copy_/comment_2_7b3f5d2e9de4b13de821177db2f57bcd._comment

diff --git a/doc/forum/more_intelligent_copy_/comment_2_7b3f5d2e9de4b13de821177db2f57bcd._comment b/doc/forum/more_intelligent_copy_/comment_2_7b3f5d2e9de4b13de821177db2f57bcd._comment
new file mode 100644
index 0000000000..474a068b65
--- /dev/null
+++ b/doc/forum/more_intelligent_copy_/comment_2_7b3f5d2e9de4b13de821177db2f57bcd._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2016-12-13T16:51:03Z"
+ content="""
+`git annex copy --fast` has the same behavior as `--not --in REMOTE`
+
+The reason this is not the default behavior is that git-annex's location
+tracking information can sometimes be out of date, and then those
+two will not copy some files despite their content not being any longer
+in the remote. This won't lead to data loss, but could result
+in unexpected behavior, and so the slower, more understandable behavior
+is used by default. (Although I sometimes go back and forth on switching
+it.)
+"""]]

From e0ab4fe748c26cd524a1a525827e84a4766d6925 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 13 Dec 2016 12:54:28 -0400
Subject: [PATCH 296/367] response

---
 .../comment_3_d74835534f52c7f123b14e5d74194733._comment    | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_3_d74835534f52c7f123b14e5d74194733._comment

diff --git a/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_3_d74835534f52c7f123b14e5d74194733._comment b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_3_d74835534f52c7f123b14e5d74194733._comment
new file mode 100644
index 0000000000..918bb4bac7
--- /dev/null
+++ b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_3_d74835534f52c7f123b14e5d74194733._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2016-12-13T16:54:11Z"
+ content="""
+Perhaps it's caused by a particular/old version of git?
+"""]]

From 279ac11ce181ac3255f81f24bf2c3c6d126ca59d Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 13 Dec 2016 12:58:22 -0400
Subject: [PATCH 297/367] response

---
 ...nt_1_ea9e3a987112d8bf6421be234bf61d3c._comment | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/forum/Multiple_interface_to_the_same_annex/comment_1_ea9e3a987112d8bf6421be234bf61d3c._comment

diff --git a/doc/forum/Multiple_interface_to_the_same_annex/comment_1_ea9e3a987112d8bf6421be234bf61d3c._comment b/doc/forum/Multiple_interface_to_the_same_annex/comment_1_ea9e3a987112d8bf6421be234bf61d3c._comment
new file mode 100644
index 0000000000..df21114bcb
--- /dev/null
+++ b/doc/forum/Multiple_interface_to_the_same_annex/comment_1_ea9e3a987112d8bf6421be234bf61d3c._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-12-13T16:55:21Z"
+ content="""
+Creating a separate special remote pointing to the same content is not a
+good idea. This will confuse git-annex's location tracking, and in some
+cases can lead to data loss, since git-annex will assume it can safely
+delete a file from one of the "two" repositories, since it thinks the
+"other" one will still have the content of the file.
+
+Instead, you need a way to configure a regular git remote,
+pointing at the repository, that is read-only, or is accessed over rsync,
+or whatever your requirements are. 
+"""]]

From a12eac060c0647e88a90c6b8dc228ef183b798bd Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 13 Dec 2016 13:59:28 -0400
Subject: [PATCH 298/367] updates

---
 doc/design/assistant/telehash.mdwn | 52 ++++++++----------------------
 doc/design/roadmap.mdwn            |  1 -
 doc/todo/tor.mdwn                  |  5 +--
 3 files changed, 17 insertions(+), 41 deletions(-)

diff --git a/doc/design/assistant/telehash.mdwn b/doc/design/assistant/telehash.mdwn
index 6a098ba515..6656ab3b22 100644
--- a/doc/design/assistant/telehash.mdwn
+++ b/doc/design/assistant/telehash.mdwn
@@ -46,13 +46,7 @@ or [cjdns](https://github.com/cjdelisle/cjdns) or tor or i2p or [magic wormhole]
 
 * Awesome.
 * Easy to install, use; very well known.
-* May need root to set up a hidden service.
-* There's been some [haskell packages developed recently](http://www.leonmergen.com/haskell/privacy/2015/05/30/on-anonymous-networking-in-haskell-announcing-tor-and-i2p-for-haskell.html) 
-  to communicate with tor and set up onion addresses for a service.
-  Could be used to make git-annex run as a hidden service.
-  However, that relies on tor being configured with a ControlPort,
-  without authentication. The normal tor configuration does not enable a
-  ControlPort.
+* Supported in git-annex now!
 
 ## i2p status
 
@@ -81,10 +75,10 @@ or [cjdns](https://github.com/cjdelisle/cjdns) or tor or i2p or [magic wormhole]
 * The remotedaemon may also support sending objects over the transport,
   depending on the transport.
 
-## address discovery
+## address exchange
 
 The address is a public key, and the authtoken is some large chunk of data,
-so won't want to type that in. Need discovery.
+so won't want to type that in. Need discovery or exchange for peering.
 
 * Easy way is any set of repos that are already connected can communicate
   them via address.log.
@@ -95,36 +89,18 @@ so won't want to type that in. Need discovery.
   it can be read over the phone.
 * Users may not have a way to communicate with perfect forward secrecy.
   So it would be good to have a address+authtoken that can only be used
-  one time during pairing:
+  one time during pairing.
+* Check out [PAKE](https://en.wikipedia.org/wiki/Password-authenticated_key_agreement)
+  for MITM resistance.
+* Possibly use magic wormhole to exchange the address, which avoids
+  the users needing to exchange so much data. The magic wormhole code
+  is just 3 words, and it uses PAKE.
+  
+  I tried it, and opened a couple of bug reports:
 
-  1. Alice uses the webapp to generate a one-time address+authtoken,
-     and sends it into a message to Bob.
-  2. Bob enters it into his webapp.
-  3. Bob's assistant contacts Alice's over the transport, presents the
-     one-time authtoken. (Alice's assistant accepts it, and marks it as
-     used so it cannot be used again.)
-  4. Alice's webapp shows that it's ready to finish pairing; so does Bob's.
-     Both wait for their users to confirm before proceeding.
-  5. Alice's assistant generates a new, permanant use authtoken, sends it
-     to Bob's assistant, which stores it and enables a remote using it.
-  6. Bob's assistant generates a new, permanant use authtoken, sends it to
-     Alice's assistant, which stores it and enables a remote using it.
-  7. Alice and Bob's assistants are now paired.
-
-  Note that this exchange can be actively MITMed. If Eve can intercept
-  Alice's message to Bob, then Eve can pair with Alice. Or, if Eve can
-  forge a message from Alice to Bob, Eve can trick Bob into pairing with
-  her.
-
-  If they make a phone call, it's much harder for Eve to MITM it.
-  Eve would need to listen to Alice reading the authtoken and enter it
-  before Bob does, so pairing with Alice. But as long as Alice waits
-  for Bob to confirm he's ready to finish pairing, this will fail,
-  because Bob won't get to that point if the authtoken is intercepted.
-
-  Check out
-  <https://en.wikipedia.org/wiki/Password-authenticated_key_agreement>
-  for more MITM resistance.
+  - [receive UI output to stdout, use stderr instead](https://github.com/warner/magic-wormhole/issues/99)
+  - [redirecting receive stdout to a pipe disables tab completion](https://github.com/warner/magic-wormhole/issues/100)
+  - [option to receive to a specific file](https://github.com/warner/magic-wormhole/issues/101)
 
 ## local lan detection
 
diff --git a/doc/design/roadmap.mdwn b/doc/design/roadmap.mdwn
index 795cc8c84e..5e636f33b9 100644
--- a/doc/design/roadmap.mdwn
+++ b/doc/design/roadmap.mdwn
@@ -3,7 +3,6 @@
 * [[design/caching_database]] for metadata views
 * [[assistant/deltas]]
 * [[assistant/gpgkeys]] management for the assistant
-* [[assistant/telehash]] or similar
 * [[design/requests_routing]]
 * [[design/new_repo_versions]]
 
diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index 01fa2635ac..262926d0fb 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -16,7 +16,8 @@ Eventually:
 * Limiting authtokens to read-only access.
 * Revoking authtokens. (This and read-only need a name associated with an
   authtoken, so the user can adjust its configuration after creating it.)
-* address exchange via electrum-mnemonic or magic wormhole (see PAKE)
-* webapp UI for easy pairing
+* address exchange for peering. See [[design/assistant/telehash]].
+* Webapp UI to set it upt.
 * friend-of-a-friend peer discovery to build more interconnected networks
   of nodes
+* Discovery of nodes on same LAN, and direct connection to them.

From 469bfa7ff3d7f213e22bde6f9c567682fa75ffa7 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 13 Dec 2016 15:35:04 -0400
Subject: [PATCH 299/367] Make all --batch input, as well as fromkey and
 registerurl stdin be processed without requiring it to be in the current
 encoding.

---
 CHANGELOG                                       |  2 ++
 CmdLine/Batch.hs                                | 17 ++++++++++-------
 Command/FromKey.hs                              |  2 +-
 Command/RegisterUrl.hs                          |  2 +-
 .../git-annex_fromkey_barfs_on_utf-8_input.mdwn |  4 ++++
 5 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index 4bd69705b0..044ab199b8 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -10,6 +10,8 @@ git-annex (6.20161211) UNRELEASED; urgency=medium
     with too many old versions of ssh and unusual ssh configurations.
     It should have not been needed anyway since ssh is supposted to
     have TCPKeepAlive enabled by default.
+  * Make all --batch input, as well as fromkey and registerurl stdin
+    be processed without requiring it to be in the current encoding.
 
  -- Joey Hess <id@joeyh.name>  Sun, 11 Dec 2016 21:29:51 -0400
 
diff --git a/CmdLine/Batch.hs b/CmdLine/Batch.hs
index 627c1df101..6ef21372f5 100644
--- a/CmdLine/Batch.hs
+++ b/CmdLine/Batch.hs
@@ -48,16 +48,19 @@ batchBadInput Batch = liftIO $ putStrLn ""
 
 -- Reads lines of batch mode input and passes to the action to handle.
 batchInput :: (String -> Either String a) -> (a -> Annex ()) -> Annex ()
-batchInput parser a = do
-	mp <- liftIO $ catchMaybeIO getLine
-	case mp of
-		Nothing -> return ()
-		Just v -> do
-			either parseerr a (parser v)
-			batchInput parser a
+batchInput parser a = go =<< batchLines
   where
+	go [] = return ()
+	go (l:rest) = do
+		either parseerr a (parser l)
+		go rest
 	parseerr s = giveup $ "Batch input parse failure: " ++ s
 
+batchLines :: Annex [String]
+batchLines = liftIO $ do
+	fileEncoding stdin
+	lines <$> getContents
+
 -- Runs a CommandStart in batch mode.
 --
 -- The batch mode user expects to read a line of output, and it's up to the
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index dca63aabe2..c1e3a79650 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -45,7 +45,7 @@ startMass = do
 	next massAdd
 
 massAdd :: CommandPerform
-massAdd = go True =<< map (separate (== ' ')) . lines <$> liftIO getContents
+massAdd = go True =<< map (separate (== ' ')) <$> batchLines
   where
 	go status [] = next $ return status
 	go status ((keyname,f):rest) | not (null keyname) && not (null f) = do
diff --git a/Command/RegisterUrl.hs b/Command/RegisterUrl.hs
index 28dd2d8c5a..008e6436c6 100644
--- a/Command/RegisterUrl.hs
+++ b/Command/RegisterUrl.hs
@@ -35,7 +35,7 @@ start [] = do
 start _ = giveup "specify a key and an url"
 
 massAdd :: CommandPerform
-massAdd = go True =<< map (separate (== ' ')) . lines <$> liftIO getContents
+massAdd = go True =<< map (separate (== ' ')) <$> batchLines
   where
 	go status [] = next $ return status
 	go status ((keyname,u):rest) | not (null keyname) && not (null u) = do
diff --git a/doc/bugs/git-annex_fromkey_barfs_on_utf-8_input.mdwn b/doc/bugs/git-annex_fromkey_barfs_on_utf-8_input.mdwn
index c1f71789b9..e544015a6b 100644
--- a/doc/bugs/git-annex_fromkey_barfs_on_utf-8_input.mdwn
+++ b/doc/bugs/git-annex_fromkey_barfs_on_utf-8_input.mdwn
@@ -32,3 +32,7 @@ Note that this is indeed valid utf-8:
 00000000  c3 a9 0a                                          |...|
 00000003
 """]]
+
+> Despite my strange inability to reproduce these, there's really only one
+> thing that can fix it, namely using fileEncoding. Now done for all batch
+> and stdin reading stuff. [[fixed|done]] I suppose. --[[Joey]]

From 55a34b49300ce60c98c04125481677186bef6c87 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 13 Dec 2016 15:47:51 -0400
Subject: [PATCH 300/367] devblog

---
 doc/devblog/day_437__catching_up.mdwn | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 doc/devblog/day_437__catching_up.mdwn

diff --git a/doc/devblog/day_437__catching_up.mdwn b/doc/devblog/day_437__catching_up.mdwn
new file mode 100644
index 0000000000..1deb54459c
--- /dev/null
+++ b/doc/devblog/day_437__catching_up.mdwn
@@ -0,0 +1,20 @@
+Quite a backlog developed in the couple of weeks I was concentrating on tor
+support. I've taken a first pass through it and fixed the most pressing
+issues now. 
+
+Most important was an ugly memory corruption problem in the GHC runtime
+system that may have led to data corruption when using git-annex with Linux
+kernels older than 4.5. All the Linux standalone builds of git-annex have
+been updated to fix that issue.
+
+Today dealt with several more things, including fixing a buggy timestamp
+issue with `metadata --batch`, reverting the ssh ServerAliveInterval
+setting (broke on too many systems with old ssh or complicated ssh
+configurations), making batch input not be rejected when it can't be decoded
+as UTF-8, and more.
+
+Also, spent some time learning a little bit about Magic Wormhole and SPAKE,
+as a way to exchange tor remote addresses. Using Magic Wormhole for that
+seems like a reasonable plan. I did file a couple bugs on it which will
+need to get fixed, and then using it is mostly a question of whether it's
+easy enough to install that git-annex can rely on it.

From 4d4229be75b96f6ef01358ae60f25d0bd668b680 Mon Sep 17 00:00:00 2001
From: marekj <marekj@web>
Date: Wed, 14 Dec 2016 06:54:24 +0000
Subject: [PATCH 301/367] Create wiki page for the different levels of control
 you can have with git annex

---
 doc/workflow.mdwn | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 doc/workflow.mdwn

diff --git a/doc/workflow.mdwn b/doc/workflow.mdwn
new file mode 100644
index 0000000000..3105c6a201
--- /dev/null
+++ b/doc/workflow.mdwn
@@ -0,0 +1,19 @@
+Git-annex supports a wide variety of workflows, a spectrum that ranges from completely automatic behavior where git-annex handles everything, through manual behavior where git-annex does only what you say, when you tell it to, down to internal behavior, where you have complete control and understand how everything is stored and exactly what changes are happening.
+
+I will proceed to summarize all of these. Your question sounds like a high-level question, so I will begin at the automatic end, hoping that this is most useful, and drill down to the low level approaches. Note, however, that this the opposite order of how git-annex was apparently developed. A list of workflows that started from manual, commandline usage would be much more intuitive, but you'd have to be willing to read the man page and wiki pages to get started, and that's pretty much what's already out there anyway.
+
+Note that for each of these levels of interaction, all the levels following will also work as well. So you can actually manually move annexed files around while the webapp is running, etc.
+
+(0) git annex webapp. This command launches a local web server which serves a graphical user interface and automatically manages git annex. It will attempt to guide you through the whole process and do everything for you. I think the intent is that no other commands are needed. This should be run on every machine that may produce file changes.
+
+(1) git annex assistant without running the webapp. You could call this the command-line version of the webapp, giving you more control over creating and connecting your repositories, and configuring how files are moved between them. The assistant, when running, will automatically watch for file changes and synchronize them to other repositories, but you must manually create the repositories and configure the rules for syncing. To create a repository, use git init and then git annex init, and then git remote add it to any other repositories. If you want more than one annex, you can add their paths to ~/.config/git-annex/autostart if you would like them to automatically begin syncing when git annex assistant --autostart is run, perhaps on boot or login. You can configure rules for where files are copied using the repository setup commands such as preferred content expressions, git annex numcopies, and git config annex.largefiles; most of the settings are accessible in one place with git annex vicfg.
+
+(2) git annex watch without running the assistant. This command is like the assistant but has no automatic network behavior, giving you complete control over when repositories are pushed and pulled, and when files are moved between systems. The repository is watched, and any file changes are added to git-annex. In order to synchronize between repositories, you must run git annex sync --content in the repository with the changes, which will merge the git history and logs with your remotes, and automatically transfer files to match your preferred and required content expressions.
+
+(3) No background processes, allowing you to decide when and what files are annexed. In order to tell git-annex to manage files, you must git annex add the files.
+
+(4) Plain git annex sync without --content, giving you fine-grained control of where copies of your files are stored. This tells git-annex to merge git histories, but it does not automatically transfer your large files between systems. To transfer files and directories, you can use git annex get, git annex drop, git annex move, and git annex copy. Git-annex will not violate a required content expression or your numcopies setting unless you pass --force, so your files are still safe. This is the workflow I mostly use, and I find it the most stable. I'm trying to migrate up to --content, but I have too many large files that haven't reached their numcopies yet for that to be effective.
+
+(5) Manual management of git history without running the syncronizer, allowing you to control precisely what is committed, what commit message is used, and how your history is merged between repositories. You must have an understanding of git, and run git commit after git annex add to store the change. You must manage the git history yourself, using git pull and git push, to synchronize repositories. You may freely use git normally side-by-side with git-annex.
+
+(6) Manual management of git annex keys, giving you control of what and where git annex stores your files under the hood, and how they are associated with your working tree, rather than using the git annex add and git annex get commands which reference files automatically. Git-annex has a variety of plumbing commands listed in the man page that let you directly store and retrieve data in an annex associated with your git repository, where every datafile is enumerated by a unique hashkey.

From 1d7aabdda094acfafd3688e25f10a7b3c56ee725 Mon Sep 17 00:00:00 2001
From: marekj <marekj@web>
Date: Wed, 14 Dec 2016 07:49:26 +0000
Subject: [PATCH 302/367] Added a comment: I took the liberty to do it

---
 .../comment_4_b6f5ce361529356a77b0e6141a62c06d._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/todo/Workflow_guide/comment_4_b6f5ce361529356a77b0e6141a62c06d._comment

diff --git a/doc/todo/Workflow_guide/comment_4_b6f5ce361529356a77b0e6141a62c06d._comment b/doc/todo/Workflow_guide/comment_4_b6f5ce361529356a77b0e6141a62c06d._comment
new file mode 100644
index 0000000000..6127e3e8d7
--- /dev/null
+++ b/doc/todo/Workflow_guide/comment_4_b6f5ce361529356a77b0e6141a62c06d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="marekj"
+ avatar="http://cdn.libravatar.org/avatar/65a60e8f5183feeeef8cef815bf73e61"
+ subject="I took the liberty to do it"
+ date="2016-12-14T07:49:26Z"
+ content="""
+I simply copied @xloem's into a new [[workflow]] page. I have been looking for such a guide myself for quite some time.
+"""]]

From d395be5b38bd74f2d125a23c4d4b561538fc33b9 Mon Sep 17 00:00:00 2001
From: alpernebbi <alpernebbi@web>
Date: Wed, 14 Dec 2016 09:30:10 +0000
Subject: [PATCH 303/367] metadata --batch UTF-8 bug

---
 ...tadata_--batch_can__39__t_parse_UTF-8.mdwn | 88 +++++++++++++++++++
 1 file changed, 88 insertions(+)
 create mode 100644 doc/bugs/Linux_standalone__39__s_metadata_--batch_can__39__t_parse_UTF-8.mdwn

diff --git a/doc/bugs/Linux_standalone__39__s_metadata_--batch_can__39__t_parse_UTF-8.mdwn b/doc/bugs/Linux_standalone__39__s_metadata_--batch_can__39__t_parse_UTF-8.mdwn
new file mode 100644
index 0000000000..73c7ae8641
--- /dev/null
+++ b/doc/bugs/Linux_standalone__39__s_metadata_--batch_can__39__t_parse_UTF-8.mdwn
@@ -0,0 +1,88 @@
+### Please describe the problem.
+I had also commented this on [[another bug|bugs/git-annex_fromkey_barfs_on_utf-8_input]], but the original issue there is fixed now.
+I tested `fromkey`, `calckey --batch`, `lookupkey --batch` (in standalone) after your fix, they work nicely.
+
+However, `git-annex metadata --batch --json` using the [[linux standalone|install/Linux_standalone]] (autobuild) still fails when it encounters UTF-8 characters (e.g. ü, ç, ä).
+Also, `git-annex metadata --json` gives `"file":"��.txt"` for `ü.txt`.
+
+This happens only in the standalone builds.
+
+### What steps will reproduce the problem?
+
+[[!format sh """
+$ .../git-annex.linux/runshell
+$ touch u.txt ü.txt
+$ git-annex add .
+
+$ git-annex metadata --batch --json
+{"file":"ü.txt"}
+git-annex: Batch input parse failure: Error in $: Failed reading: Cannot decode byte '\xb3': Data.Text.Internal.Encoding.decodeUtf8: Invalid UTF-8 stream
+
+$ git-annex metadata --batch --json
+{"file":"u.txt","fields":{"ç":["b"]}}
+git-annex: Batch input parse failure: Error in $: Failed reading: Cannot decode byte '\xb3': Data.Text.Internal.Encoding.decodeUtf8: Invalid UTF-8 stream
+
+$ git-annex metadata --batch --json
+{"file":"u.txt","fields":{"b":["ä"]}}
+git-annex: Batch input parse failure: Error in $: Failed reading: Cannot decode byte '\xb3': Data.Text.Internal.Encoding.decodeUtf8: Invalid UTF-8 stream
+
+$ git-annex metadata --json
+{"command":"metadata","note":"","success":true,"key":"SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.txt","file":"u.txt","fields":{}}
+{"command":"metadata","note":"","success":true,"key":"SHA256E-s0--e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.txt","file":"��.txt","fields":{}}
+# success, but the second line should have "file":"ü.txt"
+"""]]
+
+It's the same even if I call `.../git-annex.linux/git-annex` directly (without `runshell`)
+
+### What version of git-annex are you using? On what operating system?
+Using the Linux standalone: [git-annex-standalone-amd64.tar.gz](https://downloads.kitenet.net/git-annex/autobuild/amd64/git-annex-standalone-amd64.tar.gz) on Xubuntu 16.04
+
+[[!format sh """
+$ git-annex version
+git-annex version: 6.20161213-g55a34b493
+build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+local repository version: 5
+supported repository versions: 3 5 6
+upgrade supported from repository versions: 0 1 2 3 4 5
+operating system: linux x86_64
+"""]]
+
+### Please provide any additional information below.
+
+None of the characters I used have `\xb3` in it, but all the errors happen due to it:
+[[!format sh """
+$ .../git-annex.linux/runshell
+$ echo -n ü | xxd
+00000000: c3bc                                     ..
+$ echo -n ç | xxd
+00000000: c3a7                                     ..
+$ echo -n ä | xxd
+00000000: c3a4                                     ..
+"""]]
+
+In `runshell`, `ls` can't show UTF-8, but `git-annex status` can:
+[[!format sh """
+$ .../git-annex.linux/runshell
+$ ls
+u.txt  ??.txt
+$ git-annex status
+A u.txt
+A ü.txt
+"""]]
+
+`man` complains about locale in `runshell` as well:
+[[!format sh """
+$ .../git-annex.linux/runshell
+$ man
+man: can\'t set the locale; make sure $LC_* and $LANG are correct
+What manual page do you want?
+# I escaped that \', formatting was messy otherwise
+$ set | grep LANG
+GDM_LANG='en_GB'
+LANG='en_GB.UTF-8'
+LANGUAGE='en_GB:en'
+$ set | grep LC
+# nothing
+"""]]

From 68501b9c3ebf0e88fca36a05d6af7b1413acc840 Mon Sep 17 00:00:00 2001
From: "0xloem@0bd8a79a57e4f0dcade8fc81d162c37eae4d6730" <0xloem@web>
Date: Thu, 15 Dec 2016 21:09:52 +0000
Subject: [PATCH 304/367] Add some markup and links

---
 doc/workflow.mdwn | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/doc/workflow.mdwn b/doc/workflow.mdwn
index 3105c6a201..7401ca2326 100644
--- a/doc/workflow.mdwn
+++ b/doc/workflow.mdwn
@@ -1,19 +1,26 @@
-Git-annex supports a wide variety of workflows, a spectrum that ranges from completely automatic behavior where git-annex handles everything, through manual behavior where git-annex does only what you say, when you tell it to, down to internal behavior, where you have complete control and understand how everything is stored and exactly what changes are happening.
+Git-annex supports a wide variety of workflows, a spectrum that ranges from completely automatic behavior where git-annex handles everything, through manual behavior where git-annex does only what you say when you tell it to, all the way down to internal behavior, where you have complete control and understand how everything is stored and exactly what changes are happening.
 
-I will proceed to summarize all of these. Your question sounds like a high-level question, so I will begin at the automatic end, hoping that this is most useful, and drill down to the low level approaches. Note, however, that this the opposite order of how git-annex was apparently developed. A list of workflows that started from manual, commandline usage would be much more intuitive, but you'd have to be willing to read the man page and wiki pages to get started, and that's pretty much what's already out there anyway.
+I will proceed to summarize all of these. I will begin at the automatic end, hoping that this is most useful, and drill down to the low level approaches. Note, however, that this is the opposite order of how git-annex was apparently developed. A list of workflows that started from manual, commandline usage would be much more intuitive, but you'd have to be willing to read the man page and wiki pages to get started, and that's pretty much what's already out there anyway.
 
 Note that for each of these levels of interaction, all the levels following will also work as well. So you can actually manually move annexed files around while the webapp is running, etc.
 
-(0) git annex webapp. This command launches a local web server which serves a graphical user interface and automatically manages git annex. It will attempt to guide you through the whole process and do everything for you. I think the intent is that no other commands are needed. This should be run on every machine that may produce file changes.
+# 1. [[git annex webapp|git-annex-webapp]]
+This command launches a local web server which serves a graphical user interface and automatically manages git annex. It will attempt to guide you through the whole process and do everything for you. I think the intent is that no other commands are needed. This should be run on every machine that may produce file changes.
 
-(1) git annex assistant without running the webapp. You could call this the command-line version of the webapp, giving you more control over creating and connecting your repositories, and configuring how files are moved between them. The assistant, when running, will automatically watch for file changes and synchronize them to other repositories, but you must manually create the repositories and configure the rules for syncing. To create a repository, use git init and then git annex init, and then git remote add it to any other repositories. If you want more than one annex, you can add their paths to ~/.config/git-annex/autostart if you would like them to automatically begin syncing when git annex assistant --autostart is run, perhaps on boot or login. You can configure rules for where files are copied using the repository setup commands such as preferred content expressions, git annex numcopies, and git config annex.largefiles; most of the settings are accessible in one place with git annex vicfg.
+# 2. [[git annex assistant|git-annex-assistant]] without the webapp
+You could call this the command-line version of the webapp, giving you more control over creating and connecting your repositories, and configuring how files are moved between them. The assistant, when running, will automatically watch for file changes and synchronize them to other repositories, but you must manually create the repositories and configure the rules for syncing. To create a repository, use `git init` and then `git annex init`, and then `git remote add` it to any other repositories. If you want more than one annex, you can add their paths to `~/.config/git-annex/autostart` if you would like them to automatically begin syncing when `git annex assistant --autostart` is run, perhaps on boot or login. You can configure rules for where files are copied using the repository setup commands such as [[git annex preferred-content|git-annex-preferred-content]] to configure [[content preferences|preferred content]] for what goes where, [[`git annex numcopies`|git-annex-numcopies]] for how many [[copies]] must be kept of each file, and [[`git config annex.largefiles`|tips/largefiles]] to define small files that should be stored straight in git; most of the settings are accessible in one place with [[`git annex vicfg`|git-annex-vicfg]].
 
-(2) git annex watch without running the assistant. This command is like the assistant but has no automatic network behavior, giving you complete control over when repositories are pushed and pulled, and when files are moved between systems. The repository is watched, and any file changes are added to git-annex. In order to synchronize between repositories, you must run git annex sync --content in the repository with the changes, which will merge the git history and logs with your remotes, and automatically transfer files to match your preferred and required content expressions.
+# 3. [[git annex watch|git-annex-watch]] without the assistant
+This command is like the assistant but has no automatic network behavior, giving you complete control over when repositories are pushed and pulled, and when files are moved between systems. The local repository is watched, and any file changes are added to git-annex. In order to synchronize between repositories, you must run [[`git annex sync `|git-annex-sync]]` --content` in the repository with the changes, which will merge the git history and logs with your remotes, and automatically transfer files to match your preferred and required content expressions.
 
-(3) No background processes, allowing you to decide when and what files are annexed. In order to tell git-annex to manage files, you must git annex add the files.
+# 4. No background processes
+This allows you to decide when and what files are annexed. In order to tell git-annex to manage files, you must [[`git annex add`|git-annex-add]] the files.
 
-(4) Plain git annex sync without --content, giving you fine-grained control of where copies of your files are stored. This tells git-annex to merge git histories, but it does not automatically transfer your large files between systems. To transfer files and directories, you can use git annex get, git annex drop, git annex move, and git annex copy. Git-annex will not violate a required content expression or your numcopies setting unless you pass --force, so your files are still safe. This is the workflow I mostly use, and I find it the most stable. I'm trying to migrate up to --content, but I have too many large files that haven't reached their numcopies yet for that to be effective.
+# 5. Plain [[git annex sync|git-annex-sync]] without `--content`
+This gives you fine-grained control of where copies of your files are stored. [[`git annex sync`|git-annex-sync]] without `--content` tells git-annex to merge git histories, but it does not automatically transfer your large files between systems. To transfer files and directories, you can use [[`git annex get`|git-annex-get]], [[`git annex drop`|git-annex-drop]], [[`git annex move`|git-annex-move]], and [[`git annex copy`|git-annex-copy]]. Git-annex will not violate a required content expression or your numcopies setting unless you pass `--force`, so your files are still safe. This is the workflow I mostly use, and I find it the most stable. I'm trying to migrate up to `--content`, but I have too many large files that haven't reached their numcopies yet for that to be effective.
 
-(5) Manual management of git history without running the syncronizer, allowing you to control precisely what is committed, what commit message is used, and how your history is merged between repositories. You must have an understanding of git, and run git commit after git annex add to store the change. You must manage the git history yourself, using git pull and git push, to synchronize repositories. You may freely use git normally side-by-side with git-annex.
+# 6. Manual management of git history without the synchronizer
+This allows you to control precisely what is committed to git, what commit message is used, and how your history is merged between repositories. You must have an understanding of git, and run `git commit` after `git annex add` to store the change. You must manage the git history yourself, using `git pull` and `git push`, to synchronize repositories. You may freely use git normally side-by-side with git-annex.
 
-(6) Manual management of git annex keys, giving you control of what and where git annex stores your files under the hood, and how they are associated with your working tree, rather than using the git annex add and git annex get commands which reference files automatically. Git-annex has a variety of plumbing commands listed in the man page that let you directly store and retrieve data in an annex associated with your git repository, where every datafile is enumerated by a unique hashkey.
+# 7. Manual management of git annex keys
+This gives you control of what and where git annex stores your files under the hood, and how they are associated with your working tree, rather than using the `git annex add` and `git annex get` commands which reference files automatically. Git-annex has a variety of plumbing commands listed in the [[man page|git-annex]] that let you directly store and retrieve data in an annex associated with your git repository, where every datafile is enumerated by a unique hashkey.

From bace6c83ee8f18fa1486c9204d979c978cf50432 Mon Sep 17 00:00:00 2001
From: "0xloem@0bd8a79a57e4f0dcade8fc81d162c37eae4d6730" <0xloem@web>
Date: Thu, 15 Dec 2016 21:12:45 +0000
Subject: [PATCH 305/367] Add missing link

---
 doc/workflow.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/workflow.mdwn b/doc/workflow.mdwn
index 7401ca2326..2252608734 100644
--- a/doc/workflow.mdwn
+++ b/doc/workflow.mdwn
@@ -8,7 +8,7 @@ Note that for each of these levels of interaction, all the levels following will
 This command launches a local web server which serves a graphical user interface and automatically manages git annex. It will attempt to guide you through the whole process and do everything for you. I think the intent is that no other commands are needed. This should be run on every machine that may produce file changes.
 
 # 2. [[git annex assistant|git-annex-assistant]] without the webapp
-You could call this the command-line version of the webapp, giving you more control over creating and connecting your repositories, and configuring how files are moved between them. The assistant, when running, will automatically watch for file changes and synchronize them to other repositories, but you must manually create the repositories and configure the rules for syncing. To create a repository, use `git init` and then `git annex init`, and then `git remote add` it to any other repositories. If you want more than one annex, you can add their paths to `~/.config/git-annex/autostart` if you would like them to automatically begin syncing when `git annex assistant --autostart` is run, perhaps on boot or login. You can configure rules for where files are copied using the repository setup commands such as [[git annex preferred-content|git-annex-preferred-content]] to configure [[content preferences|preferred content]] for what goes where, [[`git annex numcopies`|git-annex-numcopies]] for how many [[copies]] must be kept of each file, and [[`git config annex.largefiles`|tips/largefiles]] to define small files that should be stored straight in git; most of the settings are accessible in one place with [[`git annex vicfg`|git-annex-vicfg]].
+You could call this the command-line version of the webapp, giving you more control over creating and connecting your repositories, and configuring how files are moved between them. The assistant, when running, will automatically watch for file changes and synchronize them to other repositories, but you must manually create the repositories and configure the rules for syncing. To create a repository, use `git init` and then [[`git annex init`|git-annex-init]], and then `git remote add` it to any other repositories. If you want more than one annex, you can add their paths to `~/.config/git-annex/autostart` if you would like them to automatically begin syncing when `git annex assistant --autostart` is run, perhaps on boot or login. You can configure rules for where files are copied using the repository setup commands such as [[git annex preferred-content|git-annex-preferred-content]] to configure [[content preferences|preferred content]] for what goes where, [[`git annex numcopies`|git-annex-numcopies]] for how many [[copies]] must be kept of each file, and [[`git config annex.largefiles`|tips/largefiles]] to define small files that should be stored straight in git; most of the settings are accessible in one place with [[`git annex vicfg`|git-annex-vicfg]].
 
 # 3. [[git annex watch|git-annex-watch]] without the assistant
 This command is like the assistant but has no automatic network behavior, giving you complete control over when repositories are pushed and pulled, and when files are moved between systems. The local repository is watched, and any file changes are added to git-annex. In order to synchronize between repositories, you must run [[`git annex sync `|git-annex-sync]]` --content` in the repository with the changes, which will merge the git history and logs with your remotes, and automatically transfer files to match your preferred and required content expressions.

From 0e24aee70a255daafbd56f790ea9dd63497c2c14 Mon Sep 17 00:00:00 2001
From: "0xloem@0bd8a79a57e4f0dcade8fc81d162c37eae4d6730" <0xloem@web>
Date: Thu, 15 Dec 2016 21:18:00 +0000
Subject: [PATCH 306/367] Link adjustments

---
 doc/workflow.mdwn | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/workflow.mdwn b/doc/workflow.mdwn
index 2252608734..ad11ec1375 100644
--- a/doc/workflow.mdwn
+++ b/doc/workflow.mdwn
@@ -5,13 +5,13 @@ I will proceed to summarize all of these. I will begin at the automatic end, hop
 Note that for each of these levels of interaction, all the levels following will also work as well. So you can actually manually move annexed files around while the webapp is running, etc.
 
 # 1. [[git annex webapp|git-annex-webapp]]
-This command launches a local web server which serves a graphical user interface and automatically manages git annex. It will attempt to guide you through the whole process and do everything for you. I think the intent is that no other commands are needed. This should be run on every machine that may produce file changes.
+The [[`git annex webapp`|git-annex-webapp]] command launches a local web server which serves a graphical user interface and automatically manages git annex. It will attempt to guide you through the whole process and do everything for you. I think the intent is that no other commands are needed. This should be run on every machine that may produce file changes.
 
 # 2. [[git annex assistant|git-annex-assistant]] without the webapp
-You could call this the command-line version of the webapp, giving you more control over creating and connecting your repositories, and configuring how files are moved between them. The assistant, when running, will automatically watch for file changes and synchronize them to other repositories, but you must manually create the repositories and configure the rules for syncing. To create a repository, use `git init` and then [[`git annex init`|git-annex-init]], and then `git remote add` it to any other repositories. If you want more than one annex, you can add their paths to `~/.config/git-annex/autostart` if you would like them to automatically begin syncing when `git annex assistant --autostart` is run, perhaps on boot or login. You can configure rules for where files are copied using the repository setup commands such as [[git annex preferred-content|git-annex-preferred-content]] to configure [[content preferences|preferred content]] for what goes where, [[`git annex numcopies`|git-annex-numcopies]] for how many [[copies]] must be kept of each file, and [[`git config annex.largefiles`|tips/largefiles]] to define small files that should be stored straight in git; most of the settings are accessible in one place with [[`git annex vicfg`|git-annex-vicfg]].
+You could call [[`git annex assistant`|git-annex-assistant]] the command-line version of the webapp, giving you more control over creating and connecting your repositories, and configuring how files are moved between them. The assistant, when running, will automatically watch for file changes and synchronize them to other repositories, but you must manually create the repositories and configure the rules for syncing. To create a repository, use `git init` and then [[`git annex init`|git-annex-init]], and then `git remote add` it to any other repositories. If you want more than one annex, you can add their paths to `~/.config/git-annex/autostart` if you would like them to automatically begin syncing when `git annex assistant --autostart` is run, perhaps on boot or login. You can configure rules for where files are copied using the repository setup commands such as [[git annex preferred-content|git-annex-preferred-content]] to configure [[content preferences|preferred content]] for what goes where, [[`git annex numcopies`|git-annex-numcopies]] for how many [[copies]] must be kept of each file, and [[`git config annex.largefiles`|tips/largefiles]] to define small files that should be stored straight in git; most of the settings are accessible in one place with [[`git annex vicfg`|git-annex-vicfg]].
 
 # 3. [[git annex watch|git-annex-watch]] without the assistant
-This command is like the assistant but has no automatic network behavior, giving you complete control over when repositories are pushed and pulled, and when files are moved between systems. The local repository is watched, and any file changes are added to git-annex. In order to synchronize between repositories, you must run [[`git annex sync `|git-annex-sync]]` --content` in the repository with the changes, which will merge the git history and logs with your remotes, and automatically transfer files to match your preferred and required content expressions.
+The [[`git annex watch`|git-annex-watch]] command is like the assistant but has no automatic network behavior, giving you complete control over when repositories are pushed and pulled, and when files are moved between systems. The local repository is watched, and any file changes are added to git-annex. In order to synchronize between repositories, you must run [[`git annex sync --content`|git-annex-sync]] in the repository with the changes, which will merge the git history and logs with your remotes, and automatically transfer files to match your preferred and required content expressions.
 
 # 4. No background processes
 This allows you to decide when and what files are annexed. In order to tell git-annex to manage files, you must [[`git annex add`|git-annex-add]] the files.

From 36f22f707352a2816120e3a8b0160bbe43f39973 Mon Sep 17 00:00:00 2001
From: "moc514@eb7af2cd9147722b29f32b6606feb2b8563dfac8" <moc514@web>
Date: Fri, 16 Dec 2016 02:08:21 +0000
Subject: [PATCH 307/367] Added a comment: Nexus 6p

---
 .../comment_2_1057c0477050e52e463c36e03fcab09d._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/bugs/android__58___cannot_link_executable/comment_2_1057c0477050e52e463c36e03fcab09d._comment

diff --git a/doc/bugs/android__58___cannot_link_executable/comment_2_1057c0477050e52e463c36e03fcab09d._comment b/doc/bugs/android__58___cannot_link_executable/comment_2_1057c0477050e52e463c36e03fcab09d._comment
new file mode 100644
index 0000000000..a8469cfe0f
--- /dev/null
+++ b/doc/bugs/android__58___cannot_link_executable/comment_2_1057c0477050e52e463c36e03fcab09d._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="moc514@eb7af2cd9147722b29f32b6606feb2b8563dfac8"
+ nickname="moc514"
+ avatar="http://cdn.libravatar.org/avatar/c8c98fc66ef014e61c163375ca9e7422"
+ subject="Nexus 6p"
+ date="2016-12-16T02:08:21Z"
+ content="""
+I also have the same issue with the Nexus 6p with 7.1.1
+"""]]

From 2c038e2a4b86a9cedb834fa59dfdf3aba888d781 Mon Sep 17 00:00:00 2001
From: binx <binx@web>
Date: Fri, 16 Dec 2016 13:57:41 +0000
Subject: [PATCH 308/367] Added a comment

---
 ..._d42def5dfc1cf814fdb07f7cf808bb12._comment | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 doc/forum/two-way_assistant_sync_with_ssh_special_remote/comment_1_d42def5dfc1cf814fdb07f7cf808bb12._comment

diff --git a/doc/forum/two-way_assistant_sync_with_ssh_special_remote/comment_1_d42def5dfc1cf814fdb07f7cf808bb12._comment b/doc/forum/two-way_assistant_sync_with_ssh_special_remote/comment_1_d42def5dfc1cf814fdb07f7cf808bb12._comment
new file mode 100644
index 0000000000..a9db035800
--- /dev/null
+++ b/doc/forum/two-way_assistant_sync_with_ssh_special_remote/comment_1_d42def5dfc1cf814fdb07f7cf808bb12._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="binx"
+ avatar="http://cdn.libravatar.org/avatar/1c2b6fe37ed500f4b72c105e42e81ba9"
+ subject="comment 1"
+ date="2016-12-16T13:57:41Z"
+ content="""
+I updated git-annex on the server. I now *sometimes* get automatic two-way syncing to happen with assistant. But it is not really consistent or reliable. One thing that seems to consistently fail is when I drag and drop a file into the annex folder on the mac. When I drag filename.txt into ~/annex, and then run git annex sync, I get the message:
+
+    $ git annex sync
+    commit  
+    On branch adjusted/master(unlocked)
+    Your branch is up-to-date with 'origin/adjusted/master(unlocked)'.
+    Untracked files:
+    	.DS_Store
+    	filename.txt
+    
+    nothing added to commit but untracked files present
+    ok
+    pull u 
+    ok
+    pull origin 
+    ok
+
+"""]]

From 76d525c4d56e5f4e020eb11357bbb39d8fc06af4 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 16 Dec 2016 15:03:20 -0400
Subject: [PATCH 309/367] update links to wormhole issues

---
 doc/design/assistant/telehash.mdwn | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/design/assistant/telehash.mdwn b/doc/design/assistant/telehash.mdwn
index 6656ab3b22..788074f961 100644
--- a/doc/design/assistant/telehash.mdwn
+++ b/doc/design/assistant/telehash.mdwn
@@ -96,11 +96,11 @@ so won't want to type that in. Need discovery or exchange for peering.
   the users needing to exchange so much data. The magic wormhole code
   is just 3 words, and it uses PAKE.
   
-  I tried it, and opened a couple of bug reports:
+  I tried it, and opened a couple of bug reports that would be useful in
+  integrating it with git-annex:
 
-  - [receive UI output to stdout, use stderr instead](https://github.com/warner/magic-wormhole/issues/99)
-  - [redirecting receive stdout to a pipe disables tab completion](https://github.com/warner/magic-wormhole/issues/100)
   - [option to receive to a specific file](https://github.com/warner/magic-wormhole/issues/101)
+  - [machine readable wormhole code ](https://github.com/warner/magic-wormhole/issues/104])
 
 ## local lan detection
 

From e67a310da1961336789474b39ec8cbeb6a7d0dec Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 16 Dec 2016 15:36:59 -0400
Subject: [PATCH 310/367] p2p: --link no longer takes a remote name, instead
 the --name option can be used.

---
 CHANGELOG                                   |  2 ++
 Command/P2P.hs                              | 36 +++++++++++++++------
 doc/git-annex-p2p.mdwn                      | 12 +++++--
 doc/tips/peer_to_peer_network_with_tor.mdwn |  2 +-
 4 files changed, 39 insertions(+), 13 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index 044ab199b8..b4659fa029 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -12,6 +12,8 @@ git-annex (6.20161211) UNRELEASED; urgency=medium
     have TCPKeepAlive enabled by default.
   * Make all --batch input, as well as fromkey and registerurl stdin
     be processed without requiring it to be in the current encoding.
+  * p2p: --link no longer takes a remote name, instead the --name
+    option can be used.
 
  -- Joey Hess <id@joeyh.name>  Sun, 11 Dec 2016 21:29:51 -0400
 
diff --git a/Command/P2P.hs b/Command/P2P.hs
index ea4dd7c656..d59d774c43 100644
--- a/Command/P2P.hs
+++ b/Command/P2P.hs
@@ -27,25 +27,43 @@ cmd = command "p2p" SectionSetup
 
 data P2POpts
 	= GenAddresses
-	| LinkRemote RemoteName
+	| LinkRemote
 
-optParser :: CmdParamsDesc -> Parser P2POpts 
-optParser _ = genaddresses <|> linkremote
+optParser :: CmdParamsDesc -> Parser (P2POpts, Maybe RemoteName)
+optParser _ = (,)
+	<$> (genaddresses <|> linkremote)
+	<*> optional name
   where
 	genaddresses = flag' GenAddresses
 		( long "gen-addresses"
 		<> help "generate addresses that allow accessing this repository over P2P networks"
 		)
-	linkremote = LinkRemote <$> strOption
+	linkremote = flag' LinkRemote
 		( long "link"
-		<> metavar paramRemote
-		<> help "specify name to use for git remote"
+		<> help "set up a P2P link to a git remote"
+		)
+	name = strOption
+		( long "name"
+		<> metavar paramName
+		<> help "name of remote"
 		)
 
-seek :: P2POpts -> CommandSeek
-seek GenAddresses = genAddresses =<< loadP2PAddresses
-seek (LinkRemote name) = commandAction $
+seek :: (P2POpts, Maybe RemoteName) -> CommandSeek
+seek (GenAddresses, _) = genAddresses =<< loadP2PAddresses
+seek (LinkRemote, Just name) = commandAction $
 	linkRemote (Git.Remote.makeLegalName name)
+seek (LinkRemote, Nothing) = commandAction $
+	linkRemote =<< unusedPeerRemoteName
+
+unusedPeerRemoteName :: Annex RemoteName
+unusedPeerRemoteName = go (1 :: Integer) =<< usednames
+  where
+	usednames = mapMaybe remoteName . remotes <$> Annex.gitRepo
+	go n names = do
+		let name = "peer" ++ show n
+		if name `elem` names
+			then go (n+1) names
+			else return name
 
 -- Only addresses are output to stdout, to allow scripting.
 genAddresses :: [P2PAddress] -> Annex ()
diff --git a/doc/git-annex-p2p.mdwn b/doc/git-annex-p2p.mdwn
index 5bf48178fe..6c50c9dd2f 100644
--- a/doc/git-annex-p2p.mdwn
+++ b/doc/git-annex-p2p.mdwn
@@ -22,14 +22,20 @@ services.
   over the available P2P networks. The address or addresses is output to
   stdout.
 
-* `--link remotename`
+* `--link`
 
-  Sets up a git remote with the specified remotename that is accessed over
-  a P2P network. 
+  Sets up a git remote that is accessed over a P2P network.
   
   This will prompt for an address to be entered; you should paste in the
   address that was generated by --gen-address in the remote repository.
 
+  Defaults to making the git remote be named "peer1", "peer2",
+  etc. This can be overridden with the `--name` option.
+
+* `--name`
+
+  Specify a name to use when setting up a git remote.
+
 # SEE ALSO
 
 [[git-annex]](1)
diff --git a/doc/tips/peer_to_peer_network_with_tor.mdwn b/doc/tips/peer_to_peer_network_with_tor.mdwn
index 11d977cca0..9c97735e43 100644
--- a/doc/tips/peer_to_peer_network_with_tor.mdwn
+++ b/doc/tips/peer_to_peer_network_with_tor.mdwn
@@ -46,7 +46,7 @@ Now, tell the new peer about the address of the first peer.
 This will make a git remote named "peer1", which connects,
 through Tor, to the repository on the other peer.
 
-	git annex p2p --link peer1
+	git annex p2p --link --name peer1
 
 That command will prompt for an address; paste in the address that was
 generated on the first peer, and then press Enter.

From 3037feb1bf9ae9c857b45191309965859b23b0b6 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 16 Dec 2016 16:32:29 -0400
Subject: [PATCH 311/367] p2p --link now defaults to setting up a
 bi-directional link

Both the local and remote git repositories get remotes added
pointing at one-another.

Makes pairing twice as easy!

Security: The new LINK command in the protocol can be sent repeatedly,
but only by a peer who has authenticated with us. So, it's entirely safe to
add a link back to that peer, or to some other peer it knows about.
Anything we receive over such a link, the peer could send us over the
current connection.

There is some risk of being flooded with LINKs, and adding too many
remotes. To guard against that, there's a hard cap on the number of remotes
that can be set up this way. This will only be a problem if setting up
large p2p networks that have exceptional interconnectedness.

A new, dedicated authtoken is created when sending LINK.

This also allows, in theory, using a p2p network like tor, to learn about
links on other networks, like telehash.

This commit was sponsored by Bruno BEAUFILS on Patreon.
---
 CHANGELOG                                   |  3 +
 Command/P2P.hs                              | 45 +++------------
 P2P/Address.hs                              |  5 ++
 P2P/Annex.hs                                | 63 +++++++++++++++++++++
 P2P/Protocol.hs                             | 16 +++++-
 doc/git-annex-p2p.mdwn                      |  9 ++-
 doc/tips/peer_to_peer_network_with_tor.mdwn | 10 ++--
 7 files changed, 105 insertions(+), 46 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index b4659fa029..faadc132e1 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -14,6 +14,9 @@ git-annex (6.20161211) UNRELEASED; urgency=medium
     be processed without requiring it to be in the current encoding.
   * p2p: --link no longer takes a remote name, instead the --name
     option can be used.
+  * p2p --link now defaults to setting up a bi-directional link;
+    both the local and remote git repositories get remotes added
+    pointing at one-another.
 
  -- Joey Hess <id@joeyh.name>  Sun, 11 Dec 2016 21:29:51 -0400
 
diff --git a/Command/P2P.hs b/Command/P2P.hs
index d59d774c43..323906f361 100644
--- a/Command/P2P.hs
+++ b/Command/P2P.hs
@@ -10,15 +10,10 @@ module Command.P2P where
 import Command
 import P2P.Address
 import P2P.Auth
-import P2P.IO
-import qualified P2P.Protocol as P2P
+import P2P.Annex
 import Utility.AuthToken
 import Git.Types
 import qualified Git.Remote
-import qualified Git.Command
-import qualified Annex
-import Annex.UUID
-import Config
 
 cmd :: Command
 cmd = command "p2p" SectionSetup
@@ -55,16 +50,6 @@ seek (LinkRemote, Just name) = commandAction $
 seek (LinkRemote, Nothing) = commandAction $
 	linkRemote =<< unusedPeerRemoteName
 
-unusedPeerRemoteName :: Annex RemoteName
-unusedPeerRemoteName = go (1 :: Integer) =<< usednames
-  where
-	usednames = mapMaybe remoteName . remotes <$> Annex.gitRepo
-	go n names = do
-		let name = "peer" ++ show n
-		if name `elem` names
-			then go (n+1) names
-			else return name
-
 -- Only addresses are output to stdout, to allow scripting.
 genAddresses :: [P2PAddress] -> Annex ()
 genAddresses [] = giveup "No P2P networks are currrently available."
@@ -95,24 +80,10 @@ linkRemote remotename = do
 				Nothing -> do
 					liftIO $ hPutStrLn stderr "Unable to parse that address, please check its format and try again."
 					prompt
-				Just addr -> setup addr
-	setup (P2PAddressAuth addr authtoken) = do
-		g <- Annex.gitRepo
-		conn <- liftIO $ connectPeer g addr
-			`catchNonAsync` connerror
-		u <- getUUID
-		v <- liftIO $ runNetProto conn $ P2P.auth u authtoken
-		case v of
-			Right (Just theiruuid) -> do
-				ok <- inRepo $ Git.Command.runBool
-					[ Param "remote", Param "add"
-					, Param remotename
-					, Param (formatP2PAddress addr)
-					]
-				when ok $ do
-					storeUUIDIn (remoteConfig remotename "uuid") theiruuid
-					storeP2PRemoteAuthToken addr authtoken
-				return ok
-			Right Nothing -> giveup "Unable to authenticate with peer. Please check the address and try again."
-			Left e -> giveup $ "Unable to authenticate with peer: " ++ e
-	connerror e = giveup $ "Unable to connect with peer. Please check that the peer is connected to the network, and try again. ("  ++ show e ++ ")"
+				Just addr -> do
+					myaddrs <- loadP2PAddresses
+					authtoken <- liftIO $ genAuthToken 128
+					storeP2PAuthToken authtoken
+					let linkbackto = map (`P2PAddressAuth` authtoken) myaddrs
+					linkAddress addr linkbackto remotename
+						>>= either giveup return
diff --git a/P2P/Address.hs b/P2P/Address.hs
index 1b1f66059e..9197393276 100644
--- a/P2P/Address.hs
+++ b/P2P/Address.hs
@@ -14,6 +14,7 @@ import Git.Types
 import Creds
 import Utility.AuthToken
 import Utility.Tor
+import qualified Utility.SimpleProtocol as Proto
 
 import qualified Data.Text as T
 
@@ -46,6 +47,10 @@ instance FormatP2PAddress P2PAddress where
 			return (TorAnnex (OnionAddress onionaddr) onionport)
 		| otherwise = Nothing
 
+instance Proto.Serializable P2PAddressAuth where
+	serialize = formatP2PAddress
+	deserialize = unformatP2PAddress
+
 torAnnexScheme :: String
 torAnnexScheme = "tor-annex:"
 
diff --git a/P2P/Annex.hs b/P2P/Annex.hs
index 9971762f59..56f94d2bb7 100644
--- a/P2P/Annex.hs
+++ b/P2P/Annex.hs
@@ -11,6 +11,8 @@ module P2P.Annex
 	( RunMode(..)
 	, P2PConnection(..)
 	, runFullProto
+	, unusedPeerRemoteName
+	, linkAddress
 	) where
 
 import Annex.Common
@@ -19,9 +21,16 @@ import Annex.Transfer
 import Annex.ChangedRefs
 import P2P.Protocol
 import P2P.IO
+import P2P.Address
+import P2P.Auth
 import Logs.Location
 import Types.NumCopies
 import Utility.Metered
+import qualified Git.Command
+import qualified Annex
+import Annex.UUID
+import Git.Types (RemoteName, remoteName, remotes)
+import Config
 
 import Control.Monad.Free
 
@@ -120,6 +129,17 @@ runLocal runmode runner a = case a of
 				Left e -> return (Left (show e))
 				Right changedrefs -> runner (next changedrefs)
 		_ -> return $ Left "change notification not available"
+	AddLinkToPeer addr next -> do
+		v <- tryNonAsync $ do
+			-- Flood protection; don't let a huge number
+			-- of peer remotes be created.
+			ns <- usedPeerRemoteNames
+			if length ns > 100
+				then return $ Right False
+				else linkAddress addr [] =<< unusedPeerRemoteName
+		case v of
+			Right (Right r) -> runner (next r)
+			_ -> runner (next False)
   where
 	transfer mk k af ta = case runmode of
 		-- Update transfer logs when serving.
@@ -152,3 +172,46 @@ runLocal runmode runner a = case a of
 				liftIO $ hSeek h AbsoluteSeek o
 			b <- liftIO $ hGetContentsMetered h p'
 			runner (sender b)
+
+unusedPeerRemoteName :: Annex RemoteName
+unusedPeerRemoteName = go (1 :: Integer) =<< usedPeerRemoteNames
+  where
+	go n names = do
+		let name = "peer" ++ show n
+		if name `elem` names
+			then go (n+1) names
+			else return name
+
+usedPeerRemoteNames :: Annex [RemoteName]
+usedPeerRemoteNames = filter ("peer" `isPrefixOf`) 
+	. mapMaybe remoteName . remotes <$> Annex.gitRepo
+
+linkAddress :: P2PAddressAuth -> [P2PAddressAuth] -> RemoteName -> Annex (Either String Bool)
+linkAddress (P2PAddressAuth addr authtoken) linkbackto remotename = do
+	g <- Annex.gitRepo
+	cv <- liftIO $ tryNonAsync $ connectPeer g addr
+	case cv of
+		Left e -> return $ Left $ "Unable to connect with peer. Please check that the peer is connected to the network, and try again. ("  ++ show e ++ ")"
+		Right conn -> do
+			u <- getUUID
+			v <- liftIO $ runNetProto conn $ do
+				authresp <- P2P.Protocol.auth u authtoken
+				lbok <- forM linkbackto $ P2P.Protocol.link
+				return (authresp, lbok)
+			case v of
+				Right (Just theiruuid, lbok) -> do
+					ok <- inRepo $ Git.Command.runBool
+						[ Param "remote", Param "add"
+						, Param remotename
+						, Param (formatP2PAddress addr)
+						]
+					when ok $ do
+						storeUUIDIn (remoteConfig remotename "uuid") theiruuid
+						storeP2PRemoteAuthToken addr authtoken
+					if not ok
+						then return $ Right False
+						else if or lbok || null linkbackto
+							then return $ Right True
+							else return $ Left "Linked with peer. However, the peer was unable to link back to us, so the link is one-way."
+				Right (Nothing, _) -> return $ Left "Unable to authenticate with peer. Please check the address and try again."
+				Left e -> return $ Left $ "Unable to authenticate with peer: " ++ e
diff --git a/P2P/Protocol.hs b/P2P/Protocol.hs
index 135409e262..c383fa966d 100644
--- a/P2P/Protocol.hs
+++ b/P2P/Protocol.hs
@@ -14,6 +14,7 @@ module P2P.Protocol where
 import qualified Utility.SimpleProtocol as Proto
 import Types.Key
 import Types.UUID
+import P2P.Address
 import Utility.AuthToken
 import Utility.Applicative
 import Utility.PartialPrelude
@@ -49,6 +50,7 @@ data Message
 	= AUTH UUID AuthToken -- uuid of the peer that is authenticating
 	| AUTH_SUCCESS UUID -- uuid of the remote peer
 	| AUTH_FAILURE
+	| LINK P2PAddressAuth -- sending an address that the peer may link to
 	| CONNECT Service
 	| CONNECTDONE ExitCode
 	| NOTIFYCHANGE
@@ -69,8 +71,9 @@ data Message
 
 instance Proto.Sendable Message where
 	formatMessage (AUTH uuid authtoken) = ["AUTH", Proto.serialize uuid, Proto.serialize authtoken]
-	formatMessage (AUTH_SUCCESS uuid) = ["AUTH-SUCCESS",  Proto.serialize uuid]
+	formatMessage (AUTH_SUCCESS uuid) = ["AUTH-SUCCESS", Proto.serialize uuid]
 	formatMessage AUTH_FAILURE = ["AUTH-FAILURE"]
+	formatMessage (LINK addr) = ["LINK", Proto.serialize addr]
 	formatMessage (CONNECT service) = ["CONNECT", Proto.serialize service]
 	formatMessage (CONNECTDONE exitcode) = ["CONNECTDONE", Proto.serialize exitcode]
 	formatMessage NOTIFYCHANGE = ["NOTIFYCHANGE"]
@@ -92,6 +95,7 @@ instance Proto.Receivable Message where
 	parseCommand "AUTH" = Proto.parse2 AUTH
 	parseCommand "AUTH-SUCCESS" = Proto.parse1 AUTH_SUCCESS
 	parseCommand "AUTH-FAILURE" = Proto.parse0 AUTH_FAILURE
+	parseCommand "LINK" = Proto.parse1 LINK
 	parseCommand "CONNECT" = Proto.parse1 CONNECT
 	parseCommand "CONNECTDONE" = Proto.parse1 CONNECTDONE
 	parseCommand "NOTIFYCHANGE" = Proto.parse0 NOTIFYCHANGE
@@ -236,6 +240,8 @@ data LocalF c
 	-- with False.
 	| WaitRefChange (ChangedRefs -> c)
 	-- ^ Waits for one or more git refs to change and returns them.
+	| AddLinkToPeer P2PAddressAuth (Bool -> c)
+	-- ^ Adds a link to a peer using the provided address.
 	deriving (Functor)
 
 type Local = Free LocalF
@@ -255,6 +261,11 @@ auth myuuid t = do
 			net $ sendMessage (ERROR "auth failed")
 			return Nothing
 
+link :: P2PAddressAuth -> Proto Bool
+link addr = do
+	net $ sendMessage (LINK addr)
+	checkSuccess
+
 checkPresent :: Key -> Proto Bool
 checkPresent key = do
 	net $ sendMessage (CHECKPRESENT key)
@@ -354,6 +365,9 @@ serveAuth myuuid = serverLoop handler
 serveAuthed :: UUID -> Proto ()
 serveAuthed myuuid = void $ serverLoop handler
   where
+	handler (LINK addr) = do
+		sendSuccess =<< local (addLinkToPeer addr)
+		return ServerContinue
 	handler (LOCKCONTENT key) = do
 		local $ tryLockContent key $ \locked -> do
 			sendSuccess locked
diff --git a/doc/git-annex-p2p.mdwn b/doc/git-annex-p2p.mdwn
index 6c50c9dd2f..cefa116d6a 100644
--- a/doc/git-annex-p2p.mdwn
+++ b/doc/git-annex-p2p.mdwn
@@ -24,13 +24,16 @@ services.
 
 * `--link`
 
-  Sets up a git remote that is accessed over a P2P network.
+  Sets up a link with a peer over the P2P network.
   
   This will prompt for an address to be entered; you should paste in the
   address that was generated by --gen-address in the remote repository.
 
-  Defaults to making the git remote be named "peer1", "peer2",
-  etc. This can be overridden with the `--name` option.
+  A git remote will be created, with a name like "peer1", "peer2"
+  by default (the `--name` option can be used to specify the name).
+
+  The link is bi-directional, so the peer will also have a git
+  remote added to it, linking back to the repository where this is run.
 
 * `--name`
 
diff --git a/doc/tips/peer_to_peer_network_with_tor.mdwn b/doc/tips/peer_to_peer_network_with_tor.mdwn
index 9c97735e43..13a7f0cc76 100644
--- a/doc/tips/peer_to_peer_network_with_tor.mdwn
+++ b/doc/tips/peer_to_peer_network_with_tor.mdwn
@@ -56,13 +56,13 @@ peer1 remote:
 
 	git annex sync --content peer1
 
-You can also generate an address for this new peer, by running `git annex
-p2p --gen-addresses`, and link other peers to that address using `git annex
-p2p --link`. It's often useful to link peers up in both directions,
-so peer1 is a remote of peer2 and peer2 is a remote of peer1.
-
 Any number of peers can be connected this way, within reason.
 
+(When the second peer links to it, the first peer also
+gets a new remote added to it, which points to the second peer.
+So, on the first peer, you can also sync with the second peer.
+The name of the that remote will be "peer1", or "peer2", etc.)
+
 ## starting git-annex remotedaemon
 
 Notice the `git annex remotedaemon` being run in the above examples.

From bd811d385394cad7fc016181f019b3a0583de7f7 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 16 Dec 2016 16:43:37 -0400
Subject: [PATCH 312/367] p2p: Added --one-way option.

This commit was sponsored by Fernando Jimenez on Patreon.
---
 CHANGELOG              |  1 +
 Command/P2P.hs         | 38 ++++++++++++++++++++++++--------------
 doc/git-annex-p2p.mdwn |  5 +++++
 3 files changed, 30 insertions(+), 14 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index faadc132e1..c4d3e27128 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -17,6 +17,7 @@ git-annex (6.20161211) UNRELEASED; urgency=medium
   * p2p --link now defaults to setting up a bi-directional link;
     both the local and remote git repositories get remotes added
     pointing at one-another.
+  * p2p: Added --one-way option.
 
  -- Joey Hess <id@joeyh.name>  Sun, 11 Dec 2016 21:29:51 -0400
 
diff --git a/Command/P2P.hs b/Command/P2P.hs
index 323906f361..817840d070 100644
--- a/Command/P2P.hs
+++ b/Command/P2P.hs
@@ -24,10 +24,13 @@ data P2POpts
 	= GenAddresses
 	| LinkRemote
 
-optParser :: CmdParamsDesc -> Parser (P2POpts, Maybe RemoteName)
-optParser _ = (,)
+data LinkDirection = BiDirectional | OneWay
+
+optParser :: CmdParamsDesc -> Parser (P2POpts, Maybe RemoteName, LinkDirection)
+optParser _ = (,,)
 	<$> (genaddresses <|> linkremote)
 	<*> optional name
+	<*> direction
   where
 	genaddresses = flag' GenAddresses
 		( long "gen-addresses"
@@ -42,13 +45,17 @@ optParser _ = (,)
 		<> metavar paramName
 		<> help "name of remote"
 		)
+	direction = flag BiDirectional OneWay
+		( long "one-way"
+		<> help "make one-way link, rather than default bi-directional link"
+		)
 
-seek :: (P2POpts, Maybe RemoteName) -> CommandSeek
-seek (GenAddresses, _) = genAddresses =<< loadP2PAddresses
-seek (LinkRemote, Just name) = commandAction $
-	linkRemote (Git.Remote.makeLegalName name)
-seek (LinkRemote, Nothing) = commandAction $
-	linkRemote =<< unusedPeerRemoteName
+seek :: (P2POpts, Maybe RemoteName, LinkDirection) -> CommandSeek
+seek (GenAddresses, _, _) = genAddresses =<< loadP2PAddresses
+seek (LinkRemote, Just name, direction) = commandAction $
+	linkRemote direction (Git.Remote.makeLegalName name)
+seek (LinkRemote, Nothing, direction) = commandAction $
+	linkRemote direction =<< unusedPeerRemoteName
 
 -- Only addresses are output to stdout, to allow scripting.
 genAddresses :: [P2PAddress] -> Annex ()
@@ -62,8 +69,8 @@ genAddresses addrs = do
 			map (`P2PAddressAuth` authtoken) addrs
 
 -- Address is read from stdin, to avoid leaking it in shell history.
-linkRemote :: RemoteName -> CommandStart
-linkRemote remotename = do
+linkRemote :: LinkDirection -> RemoteName -> CommandStart
+linkRemote direction remotename = do
 	showStart "p2p link" remotename
 	next $ next prompt
   where
@@ -81,9 +88,12 @@ linkRemote remotename = do
 					liftIO $ hPutStrLn stderr "Unable to parse that address, please check its format and try again."
 					prompt
 				Just addr -> do
-					myaddrs <- loadP2PAddresses
-					authtoken <- liftIO $ genAuthToken 128
-					storeP2PAuthToken authtoken
-					let linkbackto = map (`P2PAddressAuth` authtoken) myaddrs
+					linkbackto <- case direction of
+						OneWay -> return []
+						BiDirectional -> do
+							myaddrs <- loadP2PAddresses
+							authtoken <- liftIO $ genAuthToken 128
+							storeP2PAuthToken authtoken
+							return $ map (`P2PAddressAuth` authtoken) myaddrs
 					linkAddress addr linkbackto remotename
 						>>= either giveup return
diff --git a/doc/git-annex-p2p.mdwn b/doc/git-annex-p2p.mdwn
index cefa116d6a..dcfb36a3f5 100644
--- a/doc/git-annex-p2p.mdwn
+++ b/doc/git-annex-p2p.mdwn
@@ -39,6 +39,11 @@ services.
 
   Specify a name to use when setting up a git remote.
 
+* `--one-way`
+
+  Use with `--link` to create a one-way link with a peer, rather than the
+  default bi-directional link.
+
 # SEE ALSO
 
 [[git-annex]](1)

From 5779e31cc7a86d53f7141dc49d61a103bc55a60f Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 16 Dec 2016 17:41:08 -0400
Subject: [PATCH 313/367] devblog

---
 doc/devblog/day_438__bi-directional_p2p_links.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 doc/devblog/day_438__bi-directional_p2p_links.mdwn

diff --git a/doc/devblog/day_438__bi-directional_p2p_links.mdwn b/doc/devblog/day_438__bi-directional_p2p_links.mdwn
new file mode 100644
index 0000000000..abcbed1229
--- /dev/null
+++ b/doc/devblog/day_438__bi-directional_p2p_links.mdwn
@@ -0,0 +1,6 @@
+Improved `git annex p2p --link` to create a bi-directional link
+automatically. Bi-directional links are desirable more often than not, so
+it's the default behavior.
+
+Also continued thinking about using magic wormhole for communicating
+p2p addresses for pairing. And filed some more bugs on magic wormhole.

From 38f9337e160190823ad929cd368077338314a73c Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Fri, 16 Dec 2016 18:26:07 -0400
Subject: [PATCH 314/367] Revert "p2p --link now defaults to setting up a
 bi-directional link"

This reverts commit 3037feb1bf9ae9c857b45191309965859b23b0b6.

On second thought, this was an overcomplication of what should be the
lowest-level primitive. Let's build bi-directional links at the pairing
level with eg magic wormhole.
---
 CHANGELOG                                   |  4 --
 Command/P2P.hs                              | 75 +++++++++++++--------
 P2P/Address.hs                              |  5 --
 P2P/Annex.hs                                | 63 -----------------
 P2P/Protocol.hs                             | 16 +----
 doc/git-annex-p2p.mdwn                      | 14 +---
 doc/tips/peer_to_peer_network_with_tor.mdwn | 10 +--
 7 files changed, 56 insertions(+), 131 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index c4d3e27128..b4659fa029 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -14,10 +14,6 @@ git-annex (6.20161211) UNRELEASED; urgency=medium
     be processed without requiring it to be in the current encoding.
   * p2p: --link no longer takes a remote name, instead the --name
     option can be used.
-  * p2p --link now defaults to setting up a bi-directional link;
-    both the local and remote git repositories get remotes added
-    pointing at one-another.
-  * p2p: Added --one-way option.
 
  -- Joey Hess <id@joeyh.name>  Sun, 11 Dec 2016 21:29:51 -0400
 
diff --git a/Command/P2P.hs b/Command/P2P.hs
index 817840d070..d59d774c43 100644
--- a/Command/P2P.hs
+++ b/Command/P2P.hs
@@ -10,10 +10,15 @@ module Command.P2P where
 import Command
 import P2P.Address
 import P2P.Auth
-import P2P.Annex
+import P2P.IO
+import qualified P2P.Protocol as P2P
 import Utility.AuthToken
 import Git.Types
 import qualified Git.Remote
+import qualified Git.Command
+import qualified Annex
+import Annex.UUID
+import Config
 
 cmd :: Command
 cmd = command "p2p" SectionSetup
@@ -24,13 +29,10 @@ data P2POpts
 	= GenAddresses
 	| LinkRemote
 
-data LinkDirection = BiDirectional | OneWay
-
-optParser :: CmdParamsDesc -> Parser (P2POpts, Maybe RemoteName, LinkDirection)
-optParser _ = (,,)
+optParser :: CmdParamsDesc -> Parser (P2POpts, Maybe RemoteName)
+optParser _ = (,)
 	<$> (genaddresses <|> linkremote)
 	<*> optional name
-	<*> direction
   where
 	genaddresses = flag' GenAddresses
 		( long "gen-addresses"
@@ -45,17 +47,23 @@ optParser _ = (,,)
 		<> metavar paramName
 		<> help "name of remote"
 		)
-	direction = flag BiDirectional OneWay
-		( long "one-way"
-		<> help "make one-way link, rather than default bi-directional link"
-		)
 
-seek :: (P2POpts, Maybe RemoteName, LinkDirection) -> CommandSeek
-seek (GenAddresses, _, _) = genAddresses =<< loadP2PAddresses
-seek (LinkRemote, Just name, direction) = commandAction $
-	linkRemote direction (Git.Remote.makeLegalName name)
-seek (LinkRemote, Nothing, direction) = commandAction $
-	linkRemote direction =<< unusedPeerRemoteName
+seek :: (P2POpts, Maybe RemoteName) -> CommandSeek
+seek (GenAddresses, _) = genAddresses =<< loadP2PAddresses
+seek (LinkRemote, Just name) = commandAction $
+	linkRemote (Git.Remote.makeLegalName name)
+seek (LinkRemote, Nothing) = commandAction $
+	linkRemote =<< unusedPeerRemoteName
+
+unusedPeerRemoteName :: Annex RemoteName
+unusedPeerRemoteName = go (1 :: Integer) =<< usednames
+  where
+	usednames = mapMaybe remoteName . remotes <$> Annex.gitRepo
+	go n names = do
+		let name = "peer" ++ show n
+		if name `elem` names
+			then go (n+1) names
+			else return name
 
 -- Only addresses are output to stdout, to allow scripting.
 genAddresses :: [P2PAddress] -> Annex ()
@@ -69,8 +77,8 @@ genAddresses addrs = do
 			map (`P2PAddressAuth` authtoken) addrs
 
 -- Address is read from stdin, to avoid leaking it in shell history.
-linkRemote :: LinkDirection -> RemoteName -> CommandStart
-linkRemote direction remotename = do
+linkRemote :: RemoteName -> CommandStart
+linkRemote remotename = do
 	showStart "p2p link" remotename
 	next $ next prompt
   where
@@ -87,13 +95,24 @@ linkRemote direction remotename = do
 				Nothing -> do
 					liftIO $ hPutStrLn stderr "Unable to parse that address, please check its format and try again."
 					prompt
-				Just addr -> do
-					linkbackto <- case direction of
-						OneWay -> return []
-						BiDirectional -> do
-							myaddrs <- loadP2PAddresses
-							authtoken <- liftIO $ genAuthToken 128
-							storeP2PAuthToken authtoken
-							return $ map (`P2PAddressAuth` authtoken) myaddrs
-					linkAddress addr linkbackto remotename
-						>>= either giveup return
+				Just addr -> setup addr
+	setup (P2PAddressAuth addr authtoken) = do
+		g <- Annex.gitRepo
+		conn <- liftIO $ connectPeer g addr
+			`catchNonAsync` connerror
+		u <- getUUID
+		v <- liftIO $ runNetProto conn $ P2P.auth u authtoken
+		case v of
+			Right (Just theiruuid) -> do
+				ok <- inRepo $ Git.Command.runBool
+					[ Param "remote", Param "add"
+					, Param remotename
+					, Param (formatP2PAddress addr)
+					]
+				when ok $ do
+					storeUUIDIn (remoteConfig remotename "uuid") theiruuid
+					storeP2PRemoteAuthToken addr authtoken
+				return ok
+			Right Nothing -> giveup "Unable to authenticate with peer. Please check the address and try again."
+			Left e -> giveup $ "Unable to authenticate with peer: " ++ e
+	connerror e = giveup $ "Unable to connect with peer. Please check that the peer is connected to the network, and try again. ("  ++ show e ++ ")"
diff --git a/P2P/Address.hs b/P2P/Address.hs
index 9197393276..1b1f66059e 100644
--- a/P2P/Address.hs
+++ b/P2P/Address.hs
@@ -14,7 +14,6 @@ import Git.Types
 import Creds
 import Utility.AuthToken
 import Utility.Tor
-import qualified Utility.SimpleProtocol as Proto
 
 import qualified Data.Text as T
 
@@ -47,10 +46,6 @@ instance FormatP2PAddress P2PAddress where
 			return (TorAnnex (OnionAddress onionaddr) onionport)
 		| otherwise = Nothing
 
-instance Proto.Serializable P2PAddressAuth where
-	serialize = formatP2PAddress
-	deserialize = unformatP2PAddress
-
 torAnnexScheme :: String
 torAnnexScheme = "tor-annex:"
 
diff --git a/P2P/Annex.hs b/P2P/Annex.hs
index 56f94d2bb7..9971762f59 100644
--- a/P2P/Annex.hs
+++ b/P2P/Annex.hs
@@ -11,8 +11,6 @@ module P2P.Annex
 	( RunMode(..)
 	, P2PConnection(..)
 	, runFullProto
-	, unusedPeerRemoteName
-	, linkAddress
 	) where
 
 import Annex.Common
@@ -21,16 +19,9 @@ import Annex.Transfer
 import Annex.ChangedRefs
 import P2P.Protocol
 import P2P.IO
-import P2P.Address
-import P2P.Auth
 import Logs.Location
 import Types.NumCopies
 import Utility.Metered
-import qualified Git.Command
-import qualified Annex
-import Annex.UUID
-import Git.Types (RemoteName, remoteName, remotes)
-import Config
 
 import Control.Monad.Free
 
@@ -129,17 +120,6 @@ runLocal runmode runner a = case a of
 				Left e -> return (Left (show e))
 				Right changedrefs -> runner (next changedrefs)
 		_ -> return $ Left "change notification not available"
-	AddLinkToPeer addr next -> do
-		v <- tryNonAsync $ do
-			-- Flood protection; don't let a huge number
-			-- of peer remotes be created.
-			ns <- usedPeerRemoteNames
-			if length ns > 100
-				then return $ Right False
-				else linkAddress addr [] =<< unusedPeerRemoteName
-		case v of
-			Right (Right r) -> runner (next r)
-			_ -> runner (next False)
   where
 	transfer mk k af ta = case runmode of
 		-- Update transfer logs when serving.
@@ -172,46 +152,3 @@ runLocal runmode runner a = case a of
 				liftIO $ hSeek h AbsoluteSeek o
 			b <- liftIO $ hGetContentsMetered h p'
 			runner (sender b)
-
-unusedPeerRemoteName :: Annex RemoteName
-unusedPeerRemoteName = go (1 :: Integer) =<< usedPeerRemoteNames
-  where
-	go n names = do
-		let name = "peer" ++ show n
-		if name `elem` names
-			then go (n+1) names
-			else return name
-
-usedPeerRemoteNames :: Annex [RemoteName]
-usedPeerRemoteNames = filter ("peer" `isPrefixOf`) 
-	. mapMaybe remoteName . remotes <$> Annex.gitRepo
-
-linkAddress :: P2PAddressAuth -> [P2PAddressAuth] -> RemoteName -> Annex (Either String Bool)
-linkAddress (P2PAddressAuth addr authtoken) linkbackto remotename = do
-	g <- Annex.gitRepo
-	cv <- liftIO $ tryNonAsync $ connectPeer g addr
-	case cv of
-		Left e -> return $ Left $ "Unable to connect with peer. Please check that the peer is connected to the network, and try again. ("  ++ show e ++ ")"
-		Right conn -> do
-			u <- getUUID
-			v <- liftIO $ runNetProto conn $ do
-				authresp <- P2P.Protocol.auth u authtoken
-				lbok <- forM linkbackto $ P2P.Protocol.link
-				return (authresp, lbok)
-			case v of
-				Right (Just theiruuid, lbok) -> do
-					ok <- inRepo $ Git.Command.runBool
-						[ Param "remote", Param "add"
-						, Param remotename
-						, Param (formatP2PAddress addr)
-						]
-					when ok $ do
-						storeUUIDIn (remoteConfig remotename "uuid") theiruuid
-						storeP2PRemoteAuthToken addr authtoken
-					if not ok
-						then return $ Right False
-						else if or lbok || null linkbackto
-							then return $ Right True
-							else return $ Left "Linked with peer. However, the peer was unable to link back to us, so the link is one-way."
-				Right (Nothing, _) -> return $ Left "Unable to authenticate with peer. Please check the address and try again."
-				Left e -> return $ Left $ "Unable to authenticate with peer: " ++ e
diff --git a/P2P/Protocol.hs b/P2P/Protocol.hs
index c383fa966d..135409e262 100644
--- a/P2P/Protocol.hs
+++ b/P2P/Protocol.hs
@@ -14,7 +14,6 @@ module P2P.Protocol where
 import qualified Utility.SimpleProtocol as Proto
 import Types.Key
 import Types.UUID
-import P2P.Address
 import Utility.AuthToken
 import Utility.Applicative
 import Utility.PartialPrelude
@@ -50,7 +49,6 @@ data Message
 	= AUTH UUID AuthToken -- uuid of the peer that is authenticating
 	| AUTH_SUCCESS UUID -- uuid of the remote peer
 	| AUTH_FAILURE
-	| LINK P2PAddressAuth -- sending an address that the peer may link to
 	| CONNECT Service
 	| CONNECTDONE ExitCode
 	| NOTIFYCHANGE
@@ -71,9 +69,8 @@ data Message
 
 instance Proto.Sendable Message where
 	formatMessage (AUTH uuid authtoken) = ["AUTH", Proto.serialize uuid, Proto.serialize authtoken]
-	formatMessage (AUTH_SUCCESS uuid) = ["AUTH-SUCCESS", Proto.serialize uuid]
+	formatMessage (AUTH_SUCCESS uuid) = ["AUTH-SUCCESS",  Proto.serialize uuid]
 	formatMessage AUTH_FAILURE = ["AUTH-FAILURE"]
-	formatMessage (LINK addr) = ["LINK", Proto.serialize addr]
 	formatMessage (CONNECT service) = ["CONNECT", Proto.serialize service]
 	formatMessage (CONNECTDONE exitcode) = ["CONNECTDONE", Proto.serialize exitcode]
 	formatMessage NOTIFYCHANGE = ["NOTIFYCHANGE"]
@@ -95,7 +92,6 @@ instance Proto.Receivable Message where
 	parseCommand "AUTH" = Proto.parse2 AUTH
 	parseCommand "AUTH-SUCCESS" = Proto.parse1 AUTH_SUCCESS
 	parseCommand "AUTH-FAILURE" = Proto.parse0 AUTH_FAILURE
-	parseCommand "LINK" = Proto.parse1 LINK
 	parseCommand "CONNECT" = Proto.parse1 CONNECT
 	parseCommand "CONNECTDONE" = Proto.parse1 CONNECTDONE
 	parseCommand "NOTIFYCHANGE" = Proto.parse0 NOTIFYCHANGE
@@ -240,8 +236,6 @@ data LocalF c
 	-- with False.
 	| WaitRefChange (ChangedRefs -> c)
 	-- ^ Waits for one or more git refs to change and returns them.
-	| AddLinkToPeer P2PAddressAuth (Bool -> c)
-	-- ^ Adds a link to a peer using the provided address.
 	deriving (Functor)
 
 type Local = Free LocalF
@@ -261,11 +255,6 @@ auth myuuid t = do
 			net $ sendMessage (ERROR "auth failed")
 			return Nothing
 
-link :: P2PAddressAuth -> Proto Bool
-link addr = do
-	net $ sendMessage (LINK addr)
-	checkSuccess
-
 checkPresent :: Key -> Proto Bool
 checkPresent key = do
 	net $ sendMessage (CHECKPRESENT key)
@@ -365,9 +354,6 @@ serveAuth myuuid = serverLoop handler
 serveAuthed :: UUID -> Proto ()
 serveAuthed myuuid = void $ serverLoop handler
   where
-	handler (LINK addr) = do
-		sendSuccess =<< local (addLinkToPeer addr)
-		return ServerContinue
 	handler (LOCKCONTENT key) = do
 		local $ tryLockContent key $ \locked -> do
 			sendSuccess locked
diff --git a/doc/git-annex-p2p.mdwn b/doc/git-annex-p2p.mdwn
index dcfb36a3f5..6c50c9dd2f 100644
--- a/doc/git-annex-p2p.mdwn
+++ b/doc/git-annex-p2p.mdwn
@@ -24,26 +24,18 @@ services.
 
 * `--link`
 
-  Sets up a link with a peer over the P2P network.
+  Sets up a git remote that is accessed over a P2P network.
   
   This will prompt for an address to be entered; you should paste in the
   address that was generated by --gen-address in the remote repository.
 
-  A git remote will be created, with a name like "peer1", "peer2"
-  by default (the `--name` option can be used to specify the name).
-
-  The link is bi-directional, so the peer will also have a git
-  remote added to it, linking back to the repository where this is run.
+  Defaults to making the git remote be named "peer1", "peer2",
+  etc. This can be overridden with the `--name` option.
 
 * `--name`
 
   Specify a name to use when setting up a git remote.
 
-* `--one-way`
-
-  Use with `--link` to create a one-way link with a peer, rather than the
-  default bi-directional link.
-
 # SEE ALSO
 
 [[git-annex]](1)
diff --git a/doc/tips/peer_to_peer_network_with_tor.mdwn b/doc/tips/peer_to_peer_network_with_tor.mdwn
index 13a7f0cc76..9c97735e43 100644
--- a/doc/tips/peer_to_peer_network_with_tor.mdwn
+++ b/doc/tips/peer_to_peer_network_with_tor.mdwn
@@ -56,12 +56,12 @@ peer1 remote:
 
 	git annex sync --content peer1
 
-Any number of peers can be connected this way, within reason.
+You can also generate an address for this new peer, by running `git annex
+p2p --gen-addresses`, and link other peers to that address using `git annex
+p2p --link`. It's often useful to link peers up in both directions,
+so peer1 is a remote of peer2 and peer2 is a remote of peer1.
 
-(When the second peer links to it, the first peer also
-gets a new remote added to it, which points to the second peer.
-So, on the first peer, you can also sync with the second peer.
-The name of the that remote will be "peer1", or "peer2", etc.)
+Any number of peers can be connected this way, within reason.
 
 ## starting git-annex remotedaemon
 

From fe6f36d9f3184d8c3d517c4e3cb0b8a37b86d194 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 17 Dec 2016 16:58:05 -0400
Subject: [PATCH 315/367] magic wormhole module

This interacts with it using stdio, which is surprisingly hard.

sendFile does not currently work, due to
https://github.com/warner/magic-wormhole/issues/108

Parsing the output to find the magic code is done as robustly as
possible, and should continue to work unless wormhole radically changes
the format of its codes. Presumably it will never output something that
looks like a wormhole code before the actual wormhole code; that would
also break this. It would be better if there was a way to make
wormhole not mix the code with other output, as requested in
https://github.com/warner/magic-wormhole/issues/104

Only exchange of files/directories is supported. To exchange messages,
https://github.com/warner/magic-wormhole/issues/99 would need to be resolved.
I don't need message exchange however.
---
 Utility/MagicWormhole.hs | 112 +++++++++++++++++++++++++++++++++++++++
 git-annex.cabal          |   1 +
 2 files changed, 113 insertions(+)
 create mode 100644 Utility/MagicWormhole.hs

diff --git a/Utility/MagicWormhole.hs b/Utility/MagicWormhole.hs
new file mode 100644
index 0000000000..8a3758361e
--- /dev/null
+++ b/Utility/MagicWormhole.hs
@@ -0,0 +1,112 @@
+{- Magic Wormhole integration
+ -
+ - Copyright 2016 Joey Hess <id@joeyh.name>
+ -
+ - License: BSD-2-clause
+ -}
+
+module Utility.MagicWormHole where
+
+import Utility.Process
+import Utility.SafeCommand
+import Utility.Monad
+import Utility.Misc
+import Utility.FileSystemEncoding
+
+import System.IO
+import System.Exit
+import Control.Concurrent
+import Control.Exception
+import Data.Char
+
+-- | A Magic Wormhole code.
+type Code = String
+
+-- | Codes have the form number-word-word and may contain 2 or more words.
+validCode :: String -> Bool
+validCode s = 
+	let (n, r) = separate (== '-') s
+	    (w1, w2) = separate (== '-') r
+	in and
+		[ not (null n)
+		, all isDigit n
+		, not (null w1)
+		, not (null w2)
+		, not $ any isSpace s
+		]
+
+type CodeObserver = MVar Code
+
+type WormHoleParams = [CommandParam]
+
+mkCodeObserver :: IO CodeObserver
+mkCodeObserver = newEmptyMVar
+
+waitCode :: CodeObserver -> IO Code
+waitCode = takeMVar
+
+sendCode :: CodeObserver -> Code -> IO ()
+sendCode = putMVar
+
+-- | Sends a file. Once the send is underway, the Code will be sent to the
+-- CodeObserver.
+--
+-- Currently this has to parse the output of wormhole to find the code.
+-- To make this as robust as possible, avoids looking for any particular
+-- output strings, and only looks for the form of a wormhole code
+-- (number-word-word).
+--
+-- A request to make the code available in machine-parsable form is here:
+-- https://github.com/warner/magic-wormhole/issues/104
+--
+-- XXX This currently fails due to 
+-- https://github.com/warner/magic-wormhole/issues/108
+sendFile :: FilePath -> CodeObserver -> WormHoleParams -> IO Bool
+sendFile f o ps = runWormHoleProcess p $ \_hin hout -> do
+	fileEncoding hout
+	findcode =<< words <$> hGetContents hout
+  where
+	p = wormHoleProcess (Param "send" : ps ++ [File f])
+	findcode [] = return False
+	findcode (w:ws)
+		| validCode w = do
+			sendCode o w
+			return True
+		| otherwise = findcode ws
+
+-- | Receives a file. Once the receive is under way, the Code will be
+-- read from the CodeObserver, and fed to it on stdin.
+receiveFile :: FilePath -> CodeObserver -> WormHoleParams -> IO Bool
+receiveFile f o ps = runWormHoleProcess p $ \hin hout -> do
+	hPutStrLn hin =<< waitCode o
+	hFlush hin
+	return True
+  where
+	p = wormHoleProcess $
+		[ Param "receive"
+		, Param "--accept-file"
+		, Param "--output-file"
+		, File f
+		] ++ ps
+
+wormHoleProcess :: WormHoleParams -> CreateProcess
+wormHoleProcess = proc "wormhole" . toCommand
+
+runWormHoleProcess :: CreateProcess -> (Handle -> Handle -> IO Bool) ->  IO Bool
+runWormHoleProcess p consumer = bracketOnError setup cleanup go
+  where
+	setup = do
+		(Just hin, Just hout, Nothing, pid)
+			<- createProcess p
+				{ std_in = CreatePipe
+				, std_out = CreatePipe
+				}
+		return (hin, hout, pid)
+	cleanup (hin, hout, pid) = do
+		r <- waitForProcess pid
+		hClose hin
+		hClose hout
+		return $ case r of
+			ExitSuccess -> True
+			ExitFailure _ -> False
+	go h@(hin, hout, _) = consumer hin hout <&&> cleanup h
diff --git a/git-annex.cabal b/git-annex.cabal
index 6b81424ab5..694ab24817 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1044,6 +1044,7 @@ Executable git-annex
     Utility.LockPool.Windows
     Utility.LogFile
     Utility.Lsof
+    Utility.MagicWormHole
     Utility.Matcher
     Utility.Metered
     Utility.Misc

From 399d0f19297540312a1adc0335d8537eef814594 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 17 Dec 2016 17:28:08 -0400
Subject: [PATCH 316/367] use PYTHONUNBUFFERED to force python to use sane
 stdout buffering

Works around https://github.com/warner/magic-wormhole/issues/108

See http://stackoverflow.com/questions/107705/disable-output-buffering
for the gory details. Why a scripting language would chose a default
stdout buffering that differs between terminal and piped output, and
tends to introduce this kind of bug, I don't know.
---
 Utility/MagicWormhole.hs | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/Utility/MagicWormhole.hs b/Utility/MagicWormhole.hs
index 8a3758361e..9cf101c5e5 100644
--- a/Utility/MagicWormhole.hs
+++ b/Utility/MagicWormhole.hs
@@ -12,6 +12,7 @@ import Utility.SafeCommand
 import Utility.Monad
 import Utility.Misc
 import Utility.FileSystemEncoding
+import Utility.Env
 
 import System.IO
 import System.Exit
@@ -58,13 +59,14 @@ sendCode = putMVar
 --
 -- A request to make the code available in machine-parsable form is here:
 -- https://github.com/warner/magic-wormhole/issues/104
---
--- XXX This currently fails due to 
--- https://github.com/warner/magic-wormhole/issues/108
 sendFile :: FilePath -> CodeObserver -> WormHoleParams -> IO Bool
-sendFile f o ps = runWormHoleProcess p $ \_hin hout -> do
-	fileEncoding hout
-	findcode =<< words <$> hGetContents hout
+sendFile f o ps = do
+	-- Work around stupid stdout buffering behavior of python.
+	-- See https://github.com/warner/magic-wormhole/issues/108
+	environ <- addEntry "PYTHONUNBUFFERED" "1" <$> getEnvironment
+	runWormHoleProcess p { env = Just environ} $ \_hin hout -> do
+		fileEncoding hout
+		findcode =<< words <$> hGetContents hout
   where
 	p = wormHoleProcess (Param "send" : ps ++ [File f])
 	findcode [] = return False

From 7cddfca7995fe0f544da3425e91b9e105916ba81 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 17 Dec 2016 17:36:55 -0400
Subject: [PATCH 317/367] document a minor problem

---
 Utility/MagicWormhole.hs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/Utility/MagicWormhole.hs b/Utility/MagicWormhole.hs
index 9cf101c5e5..cc3607a31b 100644
--- a/Utility/MagicWormhole.hs
+++ b/Utility/MagicWormhole.hs
@@ -55,7 +55,10 @@ sendCode = putMVar
 -- Currently this has to parse the output of wormhole to find the code.
 -- To make this as robust as possible, avoids looking for any particular
 -- output strings, and only looks for the form of a wormhole code
--- (number-word-word).
+-- (number-word-word). 
+--
+-- Note that, if the filename looks like "foo 1-wormhole-code bar", when
+-- that is output by wormhole, it will look like it's output a wormhole code.
 --
 -- A request to make the code available in machine-parsable form is here:
 -- https://github.com/warner/magic-wormhole/issues/104

From def201960212d91a50870d46c6bb3e92c44bc30a Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 17 Dec 2016 18:25:33 -0400
Subject: [PATCH 318/367] improve types

---
 Utility/MagicWormhole.hs | 72 +++++++++++++++++++++++++++-------------
 1 file changed, 49 insertions(+), 23 deletions(-)

diff --git a/Utility/MagicWormhole.hs b/Utility/MagicWormhole.hs
index cc3607a31b..4e155e6e2a 100644
--- a/Utility/MagicWormhole.hs
+++ b/Utility/MagicWormhole.hs
@@ -5,7 +5,20 @@
  - License: BSD-2-clause
  -}
 
-module Utility.MagicWormHole where
+module Utility.MagicWormHole (
+	Code,
+	mkCode,
+	validCode,
+	CodeObserver,
+	CodeProducer,
+	mkCodeObserver,
+	mkCodeProducer,
+	waitCode,
+	sendCode,
+	WormHoleParams,
+	sendFile,
+	receiveFile,
+) where
 
 import Utility.Process
 import Utility.SafeCommand
@@ -21,7 +34,13 @@ import Control.Exception
 import Data.Char
 
 -- | A Magic Wormhole code.
-type Code = String
+newtype Code = Code String
+
+-- | Smart constructor for Code
+mkCode :: String -> Maybe Code
+mkCode s
+	| validCode s = Just (Code s)
+	| otherwise = Nothing
 
 -- | Codes have the form number-word-word and may contain 2 or more words.
 validCode :: String -> Bool
@@ -36,21 +55,27 @@ validCode s =
 		, not $ any isSpace s
 		]
 
-type CodeObserver = MVar Code
+newtype CodeObserver = CodeObserver (MVar Code)
+
+newtype CodeProducer = CodeProducer (MVar Code)
+
+mkCodeObserver :: IO CodeObserver
+mkCodeObserver = CodeObserver <$> newEmptyMVar
+
+mkCodeProducer :: IO CodeProducer
+mkCodeProducer = CodeProducer <$> newEmptyMVar
+
+waitCode :: CodeObserver -> IO Code
+waitCode (CodeObserver o) = takeMVar o
+
+sendCode :: CodeProducer -> Code -> IO ()
+sendCode (CodeProducer p) = putMVar p
 
 type WormHoleParams = [CommandParam]
 
-mkCodeObserver :: IO CodeObserver
-mkCodeObserver = newEmptyMVar
-
-waitCode :: CodeObserver -> IO Code
-waitCode = takeMVar
-
-sendCode :: CodeObserver -> Code -> IO ()
-sendCode = putMVar
-
--- | Sends a file. Once the send is underway, the Code will be sent to the
--- CodeObserver.
+-- | Sends a file. Once the send is underway, and the Code has been
+-- generated, it will be sent to the CodeObserver. (This may not happen,
+-- eg if there's a network problem).
 --
 -- Currently this has to parse the output of wormhole to find the code.
 -- To make this as robust as possible, avoids looking for any particular
@@ -63,7 +88,7 @@ sendCode = putMVar
 -- A request to make the code available in machine-parsable form is here:
 -- https://github.com/warner/magic-wormhole/issues/104
 sendFile :: FilePath -> CodeObserver -> WormHoleParams -> IO Bool
-sendFile f o ps = do
+sendFile f (CodeObserver observer) ps = do
 	-- Work around stupid stdout buffering behavior of python.
 	-- See https://github.com/warner/magic-wormhole/issues/108
 	environ <- addEntry "PYTHONUNBUFFERED" "1" <$> getEnvironment
@@ -73,17 +98,18 @@ sendFile f o ps = do
   where
 	p = wormHoleProcess (Param "send" : ps ++ [File f])
 	findcode [] = return False
-	findcode (w:ws)
-		| validCode w = do
-			sendCode o w
+	findcode (w:ws) = case mkCode w of
+		Just code -> do
+			putMVar observer code
 			return True
-		| otherwise = findcode ws
+		Nothing -> findcode ws
 
 -- | Receives a file. Once the receive is under way, the Code will be
--- read from the CodeObserver, and fed to it on stdin.
-receiveFile :: FilePath -> CodeObserver -> WormHoleParams -> IO Bool
-receiveFile f o ps = runWormHoleProcess p $ \hin hout -> do
-	hPutStrLn hin =<< waitCode o
+-- read from the CodeProducer, and fed to wormhole on stdin.
+receiveFile :: FilePath -> CodeProducer -> WormHoleParams -> IO Bool
+receiveFile f (CodeProducer producer) ps = runWormHoleProcess p $ \hin _hout -> do
+	Code c <- takeMVar producer
+	hPutStrLn hin c
 	hFlush hin
 	return True
   where

From b2b6296f9ddcbeeb4043a068bcadea250848603a Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 17 Dec 2016 18:31:19 -0400
Subject: [PATCH 319/367] make sure False is returned on error

---
 Utility/MagicWormhole.hs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Utility/MagicWormhole.hs b/Utility/MagicWormhole.hs
index 4e155e6e2a..a71cc69e07 100644
--- a/Utility/MagicWormhole.hs
+++ b/Utility/MagicWormhole.hs
@@ -124,7 +124,8 @@ wormHoleProcess :: WormHoleParams -> CreateProcess
 wormHoleProcess = proc "wormhole" . toCommand
 
 runWormHoleProcess :: CreateProcess -> (Handle -> Handle -> IO Bool) ->  IO Bool
-runWormHoleProcess p consumer = bracketOnError setup cleanup go
+runWormHoleProcess p consumer = 
+	bracketOnError setup (\v -> cleanup v <&&> return False) go
   where
 	setup = do
 		(Just hin, Just hout, Nothing, pid)

From 6402318e85a69b39fc0e2735fd23831ffd0ecc06 Mon Sep 17 00:00:00 2001
From: xloem <xloem@web>
Date: Sun, 18 Dec 2016 14:56:18 +0000
Subject: [PATCH 320/367] Added a comment: Coda

---
 .../comment_9_6c3f4d165bca7a27683df286363bc19b._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_9_6c3f4d165bca7a27683df286363bc19b._comment

diff --git a/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_9_6c3f4d165bca7a27683df286363bc19b._comment b/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_9_6c3f4d165bca7a27683df286363bc19b._comment
new file mode 100644
index 0000000000..42a14f12ad
--- /dev/null
+++ b/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_9_6c3f4d165bca7a27683df286363bc19b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="xloem"
+ avatar="http://cdn.libravatar.org/avatar/b8c087f7c5e6a9358748f0727c077f3b"
+ subject="Coda"
+ date="2016-12-18T14:56:17Z"
+ content="""
+The missing files turned up in 'lost+found' the next time I ran fsck =)
+"""]]

From ccde0932a5f5812d198fc304125b5bfc17829730 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sun, 18 Dec 2016 16:50:58 -0400
Subject: [PATCH 321/367] p2p --pair with magic wormhole (untested)

It builds. I have not tried to run it yet. :)

This commit was sponsored by Jake Vosloo on Patreon.
---
 CHANGELOG                                   |   5 +-
 Command/P2P.hs                              | 221 +++++++++++++++++---
 Utility/MagicWormhole.hs                    |  13 +-
 debian/control                              |   1 +
 doc/git-annex-p2p.mdwn                      |  26 ++-
 doc/tips/peer_to_peer_network_with_tor.mdwn |  95 ++++-----
 doc/todo/tor.mdwn                           |   4 +-
 git-annex.cabal                             |   2 +-
 8 files changed, 288 insertions(+), 79 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index b4659fa029..95d1355071 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,8 @@
 git-annex (6.20161211) UNRELEASED; urgency=medium
 
-  * Debian: Build webapp on armel.
+  * p2p --pair makes it easy to pair repositories over P2P, using
+    Magic Wormhole codes to find the other repository.
+  * Debian: Recommend magic-wormhole.
   * metadata --batch: Fix bug when conflicting metadata changes were
     made in the same batch run.
   * Pass annex.web-options to wget and curl after other options, so that
@@ -14,6 +16,7 @@ git-annex (6.20161211) UNRELEASED; urgency=medium
     be processed without requiring it to be in the current encoding.
   * p2p: --link no longer takes a remote name, instead the --name
     option can be used.
+  * Debian: Build webapp on armel.
 
  -- Joey Hess <id@joeyh.name>  Sun, 11 Dec 2016 21:29:51 -0400
 
diff --git a/Command/P2P.hs b/Command/P2P.hs
index d59d774c43..ddc6c29dfa 100644
--- a/Command/P2P.hs
+++ b/Command/P2P.hs
@@ -12,13 +12,20 @@ import P2P.Address
 import P2P.Auth
 import P2P.IO
 import qualified P2P.Protocol as P2P
-import Utility.AuthToken
 import Git.Types
 import qualified Git.Remote
 import qualified Git.Command
 import qualified Annex
 import Annex.UUID
 import Config
+import Utility.AuthToken
+import Utility.Tmp
+import Utility.FileMode
+import Utility.ThreadScheduler
+import qualified Utility.MagicWormhole as Wormhole
+
+import Control.Concurrent.Async
+import qualified Data.Text as T
 
 cmd :: Command
 cmd = command "p2p" SectionSetup
@@ -28,10 +35,11 @@ cmd = command "p2p" SectionSetup
 data P2POpts
 	= GenAddresses
 	| LinkRemote
+	| Pair
 
 optParser :: CmdParamsDesc -> Parser (P2POpts, Maybe RemoteName)
 optParser _ = (,)
-	<$> (genaddresses <|> linkremote)
+	<$> (pair <|> linkremote <|> genaddresses)
 	<*> optional name
   where
 	genaddresses = flag' GenAddresses
@@ -42,7 +50,11 @@ optParser _ = (,)
 		( long "link"
 		<> help "set up a P2P link to a git remote"
 		)
-	name = strOption
+	pair = flag' Pair
+		( long "pair"
+		<> help "pair with another repository"
+		)
+	name = Git.Remote.makeLegalName <$> strOption
 		( long "name"
 		<> metavar paramName
 		<> help "name of remote"
@@ -51,9 +63,14 @@ optParser _ = (,)
 seek :: (P2POpts, Maybe RemoteName) -> CommandSeek
 seek (GenAddresses, _) = genAddresses =<< loadP2PAddresses
 seek (LinkRemote, Just name) = commandAction $
-	linkRemote (Git.Remote.makeLegalName name)
+	linkRemote name
 seek (LinkRemote, Nothing) = commandAction $
 	linkRemote =<< unusedPeerRemoteName
+seek (Pair, Just name) = commandAction $
+	pairing name =<< loadP2PAddresses
+seek (Pair, Nothing) = commandAction $ do
+	name <- unusedPeerRemoteName
+	pairing name =<< loadP2PAddresses
 
 unusedPeerRemoteName :: Annex RemoteName
 unusedPeerRemoteName = go (1 :: Integer) =<< usednames
@@ -95,24 +112,178 @@ linkRemote remotename = do
 				Nothing -> do
 					liftIO $ hPutStrLn stderr "Unable to parse that address, please check its format and try again."
 					prompt
-				Just addr -> setup addr
-	setup (P2PAddressAuth addr authtoken) = do
-		g <- Annex.gitRepo
-		conn <- liftIO $ connectPeer g addr
-			`catchNonAsync` connerror
-		u <- getUUID
-		v <- liftIO $ runNetProto conn $ P2P.auth u authtoken
-		case v of
-			Right (Just theiruuid) -> do
-				ok <- inRepo $ Git.Command.runBool
-					[ Param "remote", Param "add"
-					, Param remotename
-					, Param (formatP2PAddress addr)
-					]
-				when ok $ do
-					storeUUIDIn (remoteConfig remotename "uuid") theiruuid
-					storeP2PRemoteAuthToken addr authtoken
-				return ok
-			Right Nothing -> giveup "Unable to authenticate with peer. Please check the address and try again."
-			Left e -> giveup $ "Unable to authenticate with peer: " ++ e
-	connerror e = giveup $ "Unable to connect with peer. Please check that the peer is connected to the network, and try again. ("  ++ show e ++ ")"
+				Just addr -> do
+					r <- setupLink remotename addr
+					case r of
+						LinkSuccess -> return True
+						ConnectionError e -> giveup e
+						AuthenticationError e -> giveup e
+
+pairing :: RemoteName -> [P2PAddress] -> CommandStart
+pairing _ [] = giveup "No P2P networks are currrently available."
+pairing remotename addrs = do
+	showStart "p2p pair" remotename
+	next $ next $ do
+		r <- wormholePairing remotename addrs ui
+		case r of
+			PairSuccess -> return True
+			SendFailed -> do
+				warning "Failed sending data to pair."
+				return False
+			ReceiveFailed -> do
+				warning "Failed receiving data from pair."
+				return False
+			LinkFailed e -> do
+				warning $ "Failed linking to pair: " ++ e
+				return False
+  where
+	ui observer producer = do
+		ourcode <- Wormhole.waitCode observer
+		putStrLn ""
+		putStrLn $ "This repository's pairing code is: " ++
+			Wormhole.fromCode ourcode
+		putStrLn ""
+		theircode <- getcode ourcode
+		Wormhole.sendCode producer theircode
+	
+	getcode ourcode = do
+		putStr "Enter the other repository's pairing code: "
+		hFlush stdout
+		fileEncoding stdin
+		l <- getLine
+		case Wormhole.toCode l of
+			Just code
+				| code /= ourcode -> return code
+				| otherwise -> do
+					putStrLn "Oops -- You entered this repository's pairing code. We need the pairing code of the *other* repository."
+					getcode ourcode
+			Nothing -> do
+				putStrLn "That does not look like a valid code. Try again..."
+				getcode ourcode
+
+-- We generate half of the authtoken; the pair will provide
+-- the other half.
+newtype HalfAuthToken = HalfAuthToken T.Text
+	deriving (Show)
+
+data PairData = PairData HalfAuthToken [P2PAddress]
+	deriving (Show)
+
+serializePairData :: PairData -> String
+serializePairData (PairData (HalfAuthToken ha) addrs) = unlines $
+	T.unpack ha : map formatP2PAddress addrs
+
+deserializePairData :: String -> Maybe PairData
+deserializePairData s = case lines s of
+	[] -> Nothing
+	(ha:l) -> do
+		addrs <- mapM unformatP2PAddress l
+		return (PairData (HalfAuthToken (T.pack ha)) addrs)
+
+data PairingResult
+	= PairSuccess
+	| SendFailed
+	| ReceiveFailed
+	| LinkFailed String
+
+wormholePairing
+	:: RemoteName
+	-> [P2PAddress]
+	-> (Wormhole.CodeObserver -> Wormhole.CodeProducer -> IO ())
+	-> Annex PairingResult
+wormholePairing remotename ouraddrs ui = do
+	ourhalf <- liftIO $ HalfAuthToken . fromAuthToken
+		<$> genAuthToken 64
+	let ourpairdata = PairData ourhalf ouraddrs
+
+	-- The magic wormhole interface only supports exchanging
+	-- files. Permissions of received files may allow others
+	-- to read them. So, set up a temp directory that only
+	-- we can read.
+	withTmpDir "pair" $ \tmp -> do
+		liftIO $ void $ tryIO $ modifyFileMode tmp $ 
+			removeModes otherGroupModes
+		let sendf = tmp </> "send"
+		let recvf = tmp </> "recv"
+		liftIO $ writeFileProtected sendf $
+			serializePairData ourpairdata
+
+		observer <- liftIO Wormhole.mkCodeObserver
+		producer <- liftIO Wormhole.mkCodeProducer
+		void $ liftIO $ async $ ui observer producer
+		(sendres, recvres) <- liftIO $
+			Wormhole.sendFile sendf observer []
+				`concurrently`
+			Wormhole.receiveFile recvf producer []
+		liftIO $ nukeFile sendf
+		if sendres /= True
+			then return SendFailed
+			else if recvres /= True
+				then return ReceiveFailed
+				else do
+					r <- liftIO $ tryIO $
+						readFileStrictAnyEncoding recvf
+					case r of
+						Left _e -> return ReceiveFailed
+						Right s -> maybe 
+							(return ReceiveFailed)
+							(finishPairing 100 remotename ourhalf)
+							(deserializePairData s)
+
+-- | Allow the peer we're pairing with to authenticate to us,
+-- using an authtoken constructed from the two HalfAuthTokens.
+-- Connect to the peer we're pairing with, and try to link to them.
+--
+-- Multiple addresses may have been received for the peer. This only
+-- makes a link to one address.
+--
+-- Since we're racing the peer as they do the same, the first try is likely
+-- to fail to authenticate. Can retry any number of times, to avoid the
+-- users needing to redo the whole process.
+finishPairing :: Int -> RemoteName -> HalfAuthToken -> PairData -> Annex PairingResult
+finishPairing retries remotename (HalfAuthToken ourhalf) (PairData (HalfAuthToken theirhalf) theiraddrs) = do
+	case (toAuthToken (ourhalf <> theirhalf), toAuthToken (theirhalf <> ourhalf)) of
+		(Just ourauthtoken, Just theirauthtoken) -> do
+			liftIO $ putStrLn $ "Successfully exchanged pairing data. Connecting to " ++ remotename ++  " ..."
+			storeP2PAuthToken ourauthtoken
+			go retries theiraddrs theirauthtoken
+		_ -> return ReceiveFailed
+  where
+	go 0 [] _ = return $ LinkFailed $ "Unable to connect to " ++ remotename ++ "."
+	go n [] theirauthtoken = do
+		liftIO $ threadDelaySeconds (Seconds 2)
+		liftIO $ putStrLn $ "Unable to connect to " ++ remotename ++ ". Retrying..."
+		go (n-1) theiraddrs theirauthtoken
+	go n (addr:rest) theirauthtoken = do
+		r <- setupLink remotename (P2PAddressAuth addr theirauthtoken)
+		case r of
+			LinkSuccess -> return PairSuccess
+			_ -> go n rest theirauthtoken
+
+data LinkResult
+	= LinkSuccess
+	| ConnectionError String
+	| AuthenticationError String
+
+setupLink :: RemoteName -> P2PAddressAuth -> Annex LinkResult
+setupLink remotename (P2PAddressAuth addr authtoken) = do
+	g <- Annex.gitRepo
+	cv <- liftIO $ tryNonAsync $ connectPeer g addr
+	case cv of
+		Left e -> return $ ConnectionError $ "Unable to connect with peer. Please check that the peer is connected to the network, and try again. ("  ++ show e ++ ")"
+		Right conn -> do
+			u <- getUUID
+			go =<< liftIO (runNetProto conn $ P2P.auth u authtoken)
+  where
+	go (Right (Just theiruuid)) = do
+		ok <- inRepo $ Git.Command.runBool
+			[ Param "remote", Param "add"
+			, Param remotename
+			, Param (formatP2PAddress addr)
+			]
+		when ok $ do
+			storeUUIDIn (remoteConfig remotename "uuid") theiruuid
+			storeP2PRemoteAuthToken addr authtoken
+		return LinkSuccess
+	go (Right Nothing) = return $ AuthenticationError "Unable to authenticate with peer. Please check the address and try again."
+	go (Left e) = return $ AuthenticationError $ "Unable to authenticate with peer: " ++ e
diff --git a/Utility/MagicWormhole.hs b/Utility/MagicWormhole.hs
index a71cc69e07..9ab8048004 100644
--- a/Utility/MagicWormhole.hs
+++ b/Utility/MagicWormhole.hs
@@ -5,9 +5,11 @@
  - License: BSD-2-clause
  -}
 
-module Utility.MagicWormHole (
+module Utility.MagicWormhole (
 	Code,
 	mkCode,
+	toCode,
+	fromCode,
 	validCode,
 	CodeObserver,
 	CodeProducer,
@@ -32,9 +34,11 @@ import System.Exit
 import Control.Concurrent
 import Control.Exception
 import Data.Char
+import Data.List
 
 -- | A Magic Wormhole code.
 newtype Code = Code String
+	deriving (Eq, Show)
 
 -- | Smart constructor for Code
 mkCode :: String -> Maybe Code
@@ -42,6 +46,13 @@ mkCode s
 	| validCode s = Just (Code s)
 	| otherwise = Nothing
 
+-- | Tries to fix up some common mistakes in a homan-entered code.
+toCode :: String -> Maybe Code
+toCode s = mkCode $ intercalate "-" $ words s
+
+fromCode :: Code -> String
+fromCode (Code s) = s
+
 -- | Codes have the form number-word-word and may contain 2 or more words.
 validCode :: String -> Bool
 validCode s = 
diff --git a/debian/control b/debian/control
index 8be9fec995..644c220355 100644
--- a/debian/control
+++ b/debian/control
@@ -112,6 +112,7 @@ Recommends:
 	git-remote-gcrypt (>= 0.20130908-6),
 	nocache,
 	aria2,
+	magic-wormhole,
 Suggests:
 	xdot,
 	bup,
diff --git a/doc/git-annex-p2p.mdwn b/doc/git-annex-p2p.mdwn
index 6c50c9dd2f..127ed9a5da 100644
--- a/doc/git-annex-p2p.mdwn
+++ b/doc/git-annex-p2p.mdwn
@@ -16,11 +16,30 @@ services.
 
 # OPTIONS
 
+* `--pair`
+
+  Run this in two repositories to pair them together over the P2P network.
+
+  This will print out a code phrase, like "3-mango-elephant", and
+  will prompt for you to enter the code phrase from the other repository.
+
+  Once code phrases have been exchanged, the two repositories will
+  be paired. A git remote will be created for the other repository,
+  with a name like "peer1".
+
+  This uses [Magic Wormhole](https://github.com/warner/magic-wormhole)
+  to verify the code phrases and securely communicate the P2P addresses of
+  the repositories, so you will need it installed on both computers that are
+  being paired.
+
 * `--gen-address`
 
   Generates addresses that can be used to access this git-annex repository
   over the available P2P networks. The address or addresses is output to
-  stdout.
+  stdout. 
+  
+  Note that anyone who knows these addresses can access your
+  repository over the P2P networks.
 
 * `--link`
 
@@ -34,7 +53,8 @@ services.
 
 * `--name`
 
-  Specify a name to use when setting up a git remote.
+  Specify a name to use when setting up a git remote with `--link`
+  or `--pair`.
 
 # SEE ALSO
 
@@ -44,6 +64,8 @@ services.
 
 [[git-annex-remotedaemon]](1)
 
+wormhole(1)
+
 # AUTHOR
 
 Joey Hess <id@joeyh.name>
diff --git a/doc/tips/peer_to_peer_network_with_tor.mdwn b/doc/tips/peer_to_peer_network_with_tor.mdwn
index 9c97735e43..b6aafa5346 100644
--- a/doc/tips/peer_to_peer_network_with_tor.mdwn
+++ b/doc/tips/peer_to_peer_network_with_tor.mdwn
@@ -1,69 +1,56 @@
 git-annex has recently gotten support for running as a
 [Tor](https://torproject.org/) hidden service. This is a nice secure
-and easy to use way to connect repositories between peers in different
-locations, without needing any central server.
+and easy to use way to connect repositories in different
+locations. No account on a central server is needed; it's peer-to-peer.
 
-## setting up the first peer
+## dependencies
 
-First, you need to get Tor installed and running. See
+To use this, you need to get Tor installed and running. See
 [their website](https://torproject.org/), or try a command like:
 
 	sudo apt-get install tor
 
-To make git-annex use Tor, run these commands in your git-annex repository:
+You also need to install [Magic Wormhole](https://github.com/warner/magic-wormhole).
 
-	sudo git annex enable-tor $(id -u)
-	git annex remotedaemon
-	git annex p2p --gen-addresses
+	sudo apt-get install magic-wormhole
 
-The p2p command will output a long address, such as:
+## pairing two repositories
 
-	tor-annex::eeaytkuhaupbarfi.onion:4412:7f53c5b65b8957ef626fd461ceaae8056e3dbc459ae715e4
+You have two git-annex repositories on different computers, and want to
+connect them together over Tor so they share their contents. Or, you and a
+friend want to connect your repositories together. Pairing is an easy way
+to accomplish this.
 
-At this point, git-annex is running as a tor hidden service, but
-it will only talk to peers who know that address.
-
-## adding additional peers
-
-To add a peer, get tor installed and running on it.
-
-	sudo apt-get install tor
-
-You need a git-annex repository on the new peer. It's fine to start
-with a new empty repository:
-
-	git init annex
-	cd annex
-	git annex init
-
-And make git-annex use Tor, by running these commands in the git-annex
-repository:
+In each git-annex repository, run these commands:
 
 	sudo git annex enable-tor $(id -u)
 	git annex remotedaemon
 
-Now, tell the new peer about the address of the first peer.
-This will make a git remote named "peer1", which connects,
-through Tor, to the repository on the other peer.
+Now git-annex is running as a Tor hidden service, but 
+it will only talk to peers after pairing with them.
 
-	git annex p2p --link --name peer1
+In both repositories, run this command:
 
-That command will prompt for an address; paste in the address that was
-generated on the first peer, and then press Enter.
+	git annex p2p --pair
 
-Now you can run any commands you normally would to sync with the
-peer1 remote:
+This will print out a code phrase, like "11-incredible-tumeric",
+and prompt for you to enter the other repository's code phrase.
 
-	git annex sync --content peer1
+Once the code phrases are exchanged, the two repositories will be securely
+connected to one-another via Tor. Each will have a git remote, with a name
+like "peer1", which connects to the other repository.
 
-You can also generate an address for this new peer, by running `git annex
-p2p --gen-addresses`, and link other peers to that address using `git annex
-p2p --link`. It's often useful to link peers up in both directions,
-so peer1 is a remote of peer2 and peer2 is a remote of peer1.
+Then, you can run commands like `git annex sync peer1 --content` to sync
+with the paired repository.
 
-Any number of peers can be connected this way, within reason.
+The Magic Wormhole code phrases used during pairing will no longer be
+useful for anything afterwards.
 
-## starting git-annex remotedaemon
+Pairing connects just two repositories, but you can repeat the process to
+pair with as many other repositories as you like, in order to build up
+larger networks of repositories.
+
+## starting git-annex remotedaemon on boot
 
 Notice the `git annex remotedaemon` being run in the above examples.
 That command runs the Tor hidden service so that other peers
@@ -72,7 +59,7 @@ can connect to your repository over Tor.
 So, you may want to arrange for the remotedaemon to be started on boot.
 You can do that with a simple cron job:
 
-	@reboot cd myannexrepo && git annex remotedaemon
+	@reboot cd ~/myannexrepo && git annex remotedaemon
 
 If you use the git-annex assistant, and have it auto-starting on boot, it
 will take care of starting the remotedaemon for you.
@@ -84,9 +71,9 @@ bandwidth to go around. So, distributing large quantities (gigabytes)
 of data over Tor may be slow, and should probably be avoided.
 
 One way to avoid sending much data over tor is to set up an encrypted
-[[special_remote|special_remotes]]. git-annex knows that Tor is rather
-expensive to use, so if a file is available on a special remote as well as
-over Tor, it will download it from the special remote.
+[[special_remote|special_remotes]] someplace. git-annex knows that Tor is
+rather expensive to use, so if a file is available on a special remote as
+well as over Tor, it will download it from the special remote.
 
 You can contribute to the Tor network by
 [running a Tor relay or bridge](https://www.torproject.org/getinvolved/relays.html.en).
@@ -115,6 +102,9 @@ When you run `git annex peer --link`, it sets up a git remote using
 the onion address, and it stashes the authentication data away in a file in
 `.git/annex/creds/`
 
+When you pair repositories, these addresses are exchanged using 
+[Magic Wormhole](https://github.com/warner/magic-wormhole).
+
 ## security
 
 Tor hidden services can be quite secure. But this doesn't mean that using
@@ -144,3 +134,14 @@ to consider:
 
 * An attacker who can connect to the git-annex Tor hidden service, even
   without authenticating, can try to perform denial of service attacks.
+
+* Magic wormhole is pretty secure, but the code phrase could be guessed
+  (unlikely) or intercepted. An attacker gets just one chance to try to enter
+  the correct code phrase, before pairing finishes. If the attacker
+  successfully guesses/intercepts both code phrases, they can MITM the
+  pairing process.
+
+  If you don't want to use magic wormhole, you can instead manually generate
+  addresses with `git annex p2p --gen-addresses` and send them over an
+  authenticated, encrypted channel (such as OTR) to a friend to add with
+  `git annex p2p --link`. This may be more secure, if you get it right.
diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index 262926d0fb..cb0bc4d410 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -16,8 +16,8 @@ Eventually:
 * Limiting authtokens to read-only access.
 * Revoking authtokens. (This and read-only need a name associated with an
   authtoken, so the user can adjust its configuration after creating it.)
-* address exchange for peering. See [[design/assistant/telehash]].
-* Webapp UI to set it upt.
+* Pairing via magic wormhole.
+* Webapp UI to set it up.
 * friend-of-a-friend peer discovery to build more interconnected networks
   of nodes
 * Discovery of nodes on same LAN, and direct connection to them.
diff --git a/git-annex.cabal b/git-annex.cabal
index 694ab24817..2f07c84377 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1044,7 +1044,7 @@ Executable git-annex
     Utility.LockPool.Windows
     Utility.LogFile
     Utility.Lsof
-    Utility.MagicWormHole
+    Utility.MagicWormhole
     Utility.Matcher
     Utility.Metered
     Utility.Misc

From 7f2e7fa271be5a1c5d1863b1c4c26dde3a1ac86d Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sun, 18 Dec 2016 17:01:15 -0400
Subject: [PATCH 322/367] check if wormhole is installed

---
 Command/P2P.hs           | 28 +++++++++++++++++++++-------
 Utility/MagicWormhole.hs |  5 +++++
 2 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/Command/P2P.hs b/Command/P2P.hs
index ddc6c29dfa..f2443e787e 100644
--- a/Command/P2P.hs
+++ b/Command/P2P.hs
@@ -67,10 +67,10 @@ seek (LinkRemote, Just name) = commandAction $
 seek (LinkRemote, Nothing) = commandAction $
 	linkRemote =<< unusedPeerRemoteName
 seek (Pair, Just name) = commandAction $
-	pairing name =<< loadP2PAddresses
+	startPairing name =<< loadP2PAddresses
 seek (Pair, Nothing) = commandAction $ do
 	name <- unusedPeerRemoteName
-	pairing name =<< loadP2PAddresses
+	startPairing name =<< loadP2PAddresses
 
 unusedPeerRemoteName :: Annex RemoteName
 unusedPeerRemoteName = go (1 :: Integer) =<< usednames
@@ -119,11 +119,23 @@ linkRemote remotename = do
 						ConnectionError e -> giveup e
 						AuthenticationError e -> giveup e
 
-pairing :: RemoteName -> [P2PAddress] -> CommandStart
-pairing _ [] = giveup "No P2P networks are currrently available."
-pairing remotename addrs = do
+startPairing :: RemoteName -> [P2PAddress] -> CommandStart
+startPairing _ [] = giveup "No P2P networks are currrently available."
+startPairing remotename addrs = do
 	showStart "p2p pair" remotename
-	next $ next $ do
+	ifM (liftIO Wormhole.isInstalled)
+		( next $ performPairing remotename addrs
+		, giveup "Magic Wormhole is not installed, and is needed for pairing. Install it from your distribution or from https://github.com/warner/magic-wormhole/"
+		) 
+
+performPairing :: RemoteName -> [P2PAddress] -> CommandPerform
+performPairing remotename addrs = do
+	-- This note is displayed mainly so when magic wormhole
+	-- complains about possible protocol mismatches or other problems,
+	-- it's clear what's doing the complaining.
+	showLongNote "Will use Magic Wormhole for pairing."
+	next $ do
+		showOutput
 		r <- wormholePairing remotename addrs ui
 		case r of
 			PairSuccess -> return True
@@ -153,7 +165,9 @@ pairing remotename addrs = do
 		l <- getLine
 		case Wormhole.toCode l of
 			Just code
-				| code /= ourcode -> return code
+				| code /= ourcode -> do
+					putStrLn "Pairing in process..."
+					return code
 				| otherwise -> do
 					putStrLn "Oops -- You entered this repository's pairing code. We need the pairing code of the *other* repository."
 					getcode ourcode
diff --git a/Utility/MagicWormhole.hs b/Utility/MagicWormhole.hs
index 9ab8048004..9a99cba33f 100644
--- a/Utility/MagicWormhole.hs
+++ b/Utility/MagicWormhole.hs
@@ -20,6 +20,7 @@ module Utility.MagicWormhole (
 	WormHoleParams,
 	sendFile,
 	receiveFile,
+	isInstalled,
 ) where
 
 import Utility.Process
@@ -28,6 +29,7 @@ import Utility.Monad
 import Utility.Misc
 import Utility.FileSystemEncoding
 import Utility.Env
+import Utility.Path
 
 import System.IO
 import System.Exit
@@ -153,3 +155,6 @@ runWormHoleProcess p consumer =
 			ExitSuccess -> True
 			ExitFailure _ -> False
 	go h@(hin, hout, _) = consumer hin hout <&&> cleanup h
+
+isInstalled :: IO Bool
+isInstalled = inPath "wormhole"

From 249ddb595398caf675636794674a46bcc279bc51 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sun, 18 Dec 2016 17:13:06 -0400
Subject: [PATCH 323/367] typo

---
 Command/P2P.hs | 4 ++--
 Utility/Tor.hs | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Command/P2P.hs b/Command/P2P.hs
index f2443e787e..a4129445bf 100644
--- a/Command/P2P.hs
+++ b/Command/P2P.hs
@@ -166,7 +166,7 @@ performPairing remotename addrs = do
 		case Wormhole.toCode l of
 			Just code
 				| code /= ourcode -> do
-					putStrLn "Pairing in process..."
+					putStrLn "Exchanging pairing data..."
 					return code
 				| otherwise -> do
 					putStrLn "Oops -- You entered this repository's pairing code. We need the pairing code of the *other* repository."
@@ -258,7 +258,7 @@ finishPairing :: Int -> RemoteName -> HalfAuthToken -> PairData -> Annex Pairing
 finishPairing retries remotename (HalfAuthToken ourhalf) (PairData (HalfAuthToken theirhalf) theiraddrs) = do
 	case (toAuthToken (ourhalf <> theirhalf), toAuthToken (theirhalf <> ourhalf)) of
 		(Just ourauthtoken, Just theirauthtoken) -> do
-			liftIO $ putStrLn $ "Successfully exchanged pairing data. Connecting to " ++ remotename ++  " ..."
+			liftIO $ putStrLn $ "Successfully exchanged pairing data. Connecting to " ++ remotename ++  "..."
 			storeP2PAuthToken ourauthtoken
 			go retries theiraddrs theirauthtoken
 		_ -> return ReceiveFailed
diff --git a/Utility/Tor.hs b/Utility/Tor.hs
index 5457a5a24b..71628084ce 100644
--- a/Utility/Tor.hs
+++ b/Utility/Tor.hs
@@ -70,7 +70,7 @@ addHiddenService uid ident = do
 			-- service and generate the hostname file for it.
 			reloaded <- anyM (uncurry boolSystem)
 				[ ("systemctl", [Param "reload", Param "tor"])
-				, ("sefvice", [Param "tor", Param "reload"])
+				, ("service", [Param "tor", Param "reload"])
 				]
 			unless reloaded $
 				giveup "failed to reload tor, perhaps the tor service is not running"

From a1d6f4f9a2ff12981a34c2dd69628943dd9b61c4 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sun, 18 Dec 2016 17:23:20 -0400
Subject: [PATCH 324/367] improve note display

---
 Command/P2P.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/P2P.hs b/Command/P2P.hs
index a4129445bf..441b8a5934 100644
--- a/Command/P2P.hs
+++ b/Command/P2P.hs
@@ -133,7 +133,7 @@ performPairing remotename addrs = do
 	-- This note is displayed mainly so when magic wormhole
 	-- complains about possible protocol mismatches or other problems,
 	-- it's clear what's doing the complaining.
-	showLongNote "Will use Magic Wormhole for pairing."
+	showNote "using Magic Wormhole for pairing"
 	next $ do
 		showOutput
 		r <- wormholePairing remotename addrs ui

From 50e2d97847374cee354eae75d60c539acbb06053 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sun, 18 Dec 2016 17:31:02 -0400
Subject: [PATCH 325/367] shorten note

---
 Command/P2P.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/P2P.hs b/Command/P2P.hs
index 441b8a5934..afa5f9dc68 100644
--- a/Command/P2P.hs
+++ b/Command/P2P.hs
@@ -133,7 +133,7 @@ performPairing remotename addrs = do
 	-- This note is displayed mainly so when magic wormhole
 	-- complains about possible protocol mismatches or other problems,
 	-- it's clear what's doing the complaining.
-	showNote "using Magic Wormhole for pairing"
+	showNote "using Magic Wormhole"
 	next $ do
 		showOutput
 		r <- wormholePairing remotename addrs ui

From dc659c87b42de0d3b10cdebf85134647a9fb7882 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sun, 18 Dec 2016 17:44:37 -0400
Subject: [PATCH 326/367] devbog

---
 doc/devblog/day_439__wormhole_pairing.mdwn | 51 ++++++++++++++++++++++
 1 file changed, 51 insertions(+)
 create mode 100644 doc/devblog/day_439__wormhole_pairing.mdwn

diff --git a/doc/devblog/day_439__wormhole_pairing.mdwn b/doc/devblog/day_439__wormhole_pairing.mdwn
new file mode 100644
index 0000000000..cc988a2db3
--- /dev/null
+++ b/doc/devblog/day_439__wormhole_pairing.mdwn
@@ -0,0 +1,51 @@
+`git annex p2p --pair` implemented, using Magic Wormhole codes
+that have to be exchanged between the repositories being paired.
+
+It looks like this, with the same thing being done at the same time
+in the other repository.
+
+	joey@elephant:~/tmp/bench3/a>git annex p2p --pair
+	p2p pair peer1 (using Magic Wormhole) 
+	
+	This repository's pairing code is: 1-select-bluebird
+	
+	Enter the other repository's pairing code: (here I entered 8-fascinate-sawdust) 
+	Exchanging pairing data...
+	Successfully exchanged pairing data. Connecting to peer1...
+	ok
+
+And just that simply, the two repositories find one another, 
+Tor onion addresses and authentication data is exchanged, and a git remote
+is set up connecting via Tor.
+
+	joey@elephant:~/tmp/bench3/a>git annex sync peer1
+	commit  
+	ok
+	pull peer1 
+	warning: no common commits
+	remote: Counting objects: 5, done.
+	remote: Compressing objects: 100% (3/3), done.
+	remote: Total 5 (delta 0), reused 0 (delta 0)
+	Unpacking objects: 100% (5/5), done.
+	From tor-annex::5vkpoyz723otbmzo.onion:61900
+	 * [new branch]      git-annex  -> peer1/git-annex
+
+Very pleased with this, and also the whole thing worked on the very first
+try!
+
+It might be slightly annoying to have to exchange two codes during pairing.
+It would be possible to make this work with only one code. I decided to go
+with two codes, even though it's only marginally more secure than one,
+mostly for UI reasons. The pairing interface and 
+[[instructions for using it|tips/peer_to_peer_network_with_tor]] is simplfied
+by being symmetric.
+
+(I also decided to revert the work I did on Friday to make `p2p --link`
+set up a bidirectional link. Better to keep `--link` the simplest possible
+primitive, and pairing makes bidirectional links more easily.)
+
+Next: Some more testing of this and the Tor hidden services, a webapp UI
+for P2P peering, and then finally removing XMPP support. I hope to finish
+that by New Years.
+
+Today's work was sponsored by Jake Vosloo on Patreon.

From 3ae91fbddc8cae6c54e1801f8dcd9284ad65d734 Mon Sep 17 00:00:00 2001
From: ddenis <ddenis@web>
Date: Sun, 18 Dec 2016 23:38:30 +0000
Subject: [PATCH 327/367]

---
 ..._but_not_annex__41___controlled_files.mdwn | 102 ++++++++++++++++++
 1 file changed, 102 insertions(+)
 create mode 100644 doc/bugs/Corrupted_git___40__but_not_annex__41___controlled_files.mdwn

diff --git a/doc/bugs/Corrupted_git___40__but_not_annex__41___controlled_files.mdwn b/doc/bugs/Corrupted_git___40__but_not_annex__41___controlled_files.mdwn
new file mode 100644
index 0000000000..8f19956979
--- /dev/null
+++ b/doc/bugs/Corrupted_git___40__but_not_annex__41___controlled_files.mdwn
@@ -0,0 +1,102 @@
+### Please describe the problem.
+
+I have files that match annex.largefiles and therefore should be added to git but not to annex, they seem to be getting corrupted after cloning the repo.
+
+### What steps will reproduce the problem?
+
+I couldn't immediately find the exact steps to reproduce the issue but I have multiple git repositories showing this.
+
+### What version of git-annex are you using? On what operating system?
+
+The problem has occurred a while ago but I have just noticed it. This is on macOS if it helps. I also tend to use the latest released version of git-annex (installed via Homebrew)
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+$ cd Documents
+$ cat .gitattributes
+* annex.largefiles=((not(mimetype=text/*))or(largerthan=100kb))
+
+*.png binary
+*.jpg binary
+*.jpeg binary
+*.gif binary
+*.ico binary
+
+*.mp3 binary
+*.fla binary
+
+*.mov binary
+*.mp4 binary
+*.flv binary
+*.swf binary
+*.avi binary
+*.mkv binary
+*.mpg binary
+*.mpeg binary
+
+*.gz binary
+*.zip binary
+*.7z binary
+*.rar binary
+*.bz2 binary
+
+*.ttf binary
+
+*.pdf binary
+
+$ ls -la Docs/2016-XXX/XXX/
+total 696
+drwx------@ 4 denis  staff     136 Jul 11 15:05 ./
+drwxr-xr-x@ 9 denis  staff     306 Dec 12 19:42 ../
+-rwxr-xr-x@ 1 denis  staff  265898 Jul 11 13:03 XXX.pdf*
+-rwxr-xr-x@ 1 denis  staff   89586 Jul 11 13:03 Summary.pdf*
+$ file --mime-type Docs/2016-XXX/XXX/XXX.pdf
+Docs/2016-XXX/XXX/XXX.pdf: application/pdf
+$ git show 60a76858a57a73967131b929af45a99703f67335
+commit 60a76858a57a73967131b929af45a99703f67335
+Author: Denis Dzyubenko <denis@ddenis.info>
+Date:   Mon Jul 11 15:05:37 2016 +0200
+
+    XXX
+
+diff --git a/Docs/2016-XXX/XXX/XXX.pdf b/Docs/2016-XXX/XXX/XXX.pdf
+new file mode 100755
+index 00000000..112f68d0
+Binary files /dev/null and b/Docs/2016-XXX/XXX/XXX.pdf differ
+diff --git a/Docs/2016-XXX/XXX/Summary.pdf b/Docs/2016-XXX/XXX/Summary.pdf
+new file mode 100755
+index 00000000..3828383e
+Binary files /dev/null and b/Docs/2016-XXX/XXX/Summary.pdf differ
+diff --git a/Docs/2016-XXX/XXX.pdf b/Docs/2016-XXX/XXX.pdf
+deleted file mode 120000
+index 6d347a22..00000000
+--- a/Docs/2016-XXX/XXX.pdf
++++ /dev/null
+@@ -1 +0,0 @@
+-../../.git/annex/objects/zJ/X1/SHA256E-s190749--ee0c8329c88f9c1656cc75cf37d4df64060a022e73d199164c5e5222ba1739d1.pdf/SHA256E-s190749--ee0c8329c88f9c1656cc
+\ No newline at end of file
+
+
+
+$ git clone Documents Documents.tmp
+Cloning into 'Documents.tmp'...
+done.
+$ cd ./Documents.tmp/
+$ ls -la Docs/2016-XXX/XXX/
+total 184
+drwxr-xr-x  4 denis  staff    136 Dec 19 00:09 ./
+drwxr-xr-x  8 denis  staff    272 Dec 19 00:09 ../
+-rwxr-xr-x  1 denis  staff    101 Dec 19 00:09 XXX.pdf*
+-rwxr-xr-x  1 denis  staff  89586 Dec 19 00:09 Summary.pdf*
+$ cat Docs/2016-XXX/XXX/XXX.pdf
+/annex/objects/SHA256E-s265898--9c750c01dce9689ac3880224d2e95da6287b0cc89759c0c882e7a9a0fe48d664.pdf
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+

From 2808d11d95b5e905d1676d0b1e60a93757a78802 Mon Sep 17 00:00:00 2001
From: "m8r-achx62@7323980ed426b7f78c85dfefe7358672bce44e98" <m8r-achx62@web>
Date: Mon, 19 Dec 2016 01:43:53 +0000
Subject: [PATCH 328/367]

---
 doc/bugs/YouTube_-_error_in_importfeed.mdwn | 71 +++++++++++++++++++++
 1 file changed, 71 insertions(+)
 create mode 100644 doc/bugs/YouTube_-_error_in_importfeed.mdwn

diff --git a/doc/bugs/YouTube_-_error_in_importfeed.mdwn b/doc/bugs/YouTube_-_error_in_importfeed.mdwn
new file mode 100644
index 0000000000..b02348f656
--- /dev/null
+++ b/doc/bugs/YouTube_-_error_in_importfeed.mdwn
@@ -0,0 +1,71 @@
+### Please describe the problem.
+When adding a YouTube channel via importfeed I get the error:
+
+```
+  warning: bad feed content; no enclosures to download
+```
+
+### What steps will reproduce the problem?
+1. `cd $(mktemp -d)`
+2. `git init && git annex init`
+3. `git annex importfeed https://www.youtube.com/feeds/videos.xml\?playlist_id\=PLoXkGkpREHNBY9KtkdqhBIfx-waIGSKet`
+4. Get sad.  :-(
+
+(URL [https://www.youtube.com/feeds/videos.xml?playlist_id=PLoXkGkpREHNBY9KtkdqhBIfx-waIGSKet](https://www.youtube.com/feeds/videos.xml?playlist_id=PLoXkGkpREHNBY9KtkdqhBIfx-waIGSKet) looks like a feed to Firefox)
+
+
+### What version of git-annex are you using? On what operating system?
+OSX (MacOS?) - installed via homebrew
+
+    git-annex version: 6.20161210
+    build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV FsEvents XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+    remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+
+Debian Jessie - installed via apt-get (ASIDE: why is the apt-get version sooooo old?)
+
+    git-annex version: 5.20141125
+    build flags: Assistant Webapp Webapp-secure Pairing Testsuite S3 WebDAV Inotify DBus DesktopNotify XMPP DNS Feeds Quvi TDFA CryptoHash
+    key/value backends: SHA256E SHA1E SHA512E SHA224E SHA384E SKEIN256E SKEIN512E SHA256 SHA1 SHA512 SHA224 SHA384 SKEIN256 SKEIN512 WORM URL
+    remote types: git gcrypt S3 bup directory rsync web webdav tahoe glacier ddar hook external
+
+
+### Additional information
+
+Running with `--debug` (see below) seems to indicate that the feed downloads correctly, but it is the parsing that is failing.  I don't know what command is being run to parse the feed though.
+
+
+``` shell
+git annex importfeed --debug https://www.youtube.com/feeds/videos.xml\?playlist_id\=PLoXkGkpREHNBY9KtkdqhBIfx-waIGSKet
+```
+results in:
+
+``` shell
+(checking known urls...) [2016-12-19 12:39:36.387714] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","git-annex"]
+[2016-12-19 12:39:36.392367] process done ExitSuccess
+[2016-12-19 12:39:36.392496] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","show-ref","--hash","refs/heads/git-annex"]
+[2016-12-19 12:39:36.396484] process done ExitSuccess
+[2016-12-19 12:39:36.406716] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","ls-files","--stage","-z","--","."]
+[2016-12-19 12:39:36.412674] process done ExitSuccess
+importfeed https://www.youtube.com/feeds/videos.xml?playlist_id=PLoXkGkpREHNBY9KtkdqhBIfx-waIGSKet
+[2016-12-19 12:39:36.413555] call: wget ["--clobber","-c","-O","/var/folders/l0/l60294_970b9fh46062znm0r0000gn/T/feed16807282475249","https://www.youtube.com/feeds/videos.xml?playlist_id=PLoXkGkpREHNBY9KtkdqhBIfx-waIGSKet","--user-agent","git-annex/6.20161210"]
+--2016-12-19 12:39:36--  https://www.youtube.com/feeds/videos.xml?playlist_id=PLoXkGkpREHNBY9KtkdqhBIfx-waIGSKet
+Resolving www.youtube.com... 216.58.199.78, 2404:6800:4006:806::200e
+Connecting to www.youtube.com|216.58.199.78|:443... connected.
+HTTP request sent, awaiting response... 200 OK
+Length: unspecified [text/xml]
+Saving to: ‘/var/folders/l0/l60294_970b9fh46062znm0r0000gn/T/feed16807282475249’
+
+/var/folders/l0/l60294_970b9fh46062znm0r0000gn/T/f     [ <=>                                                                                                             ]  23.81K  --.-KB/s    in 0.02s
+
+2016-12-19 12:39:37 (1.22 MB/s) - ‘/var/folders/l0/l60294_970b9fh46062znm0r0000gn/T/feed16807282475249’ saved [24386]
+
+[2016-12-19 12:39:37.595869] process done ExitSuccess
+
+  warning: bad feed content; no enclosures to download
+ok
+```
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Yes, for years.  I donated to fund the dev and proudly display my git-annex stickers!

From dc7fda2ec57daf92802503c7c3ae599ee0fd0598 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 19 Dec 2016 12:27:16 -0400
Subject: [PATCH 329/367] section on safe pairing code exchange

---
 doc/tips/peer_to_peer_network_with_tor.mdwn | 27 ++++++++++++++++-----
 1 file changed, 21 insertions(+), 6 deletions(-)

diff --git a/doc/tips/peer_to_peer_network_with_tor.mdwn b/doc/tips/peer_to_peer_network_with_tor.mdwn
index b6aafa5346..a57d8a5449 100644
--- a/doc/tips/peer_to_peer_network_with_tor.mdwn
+++ b/doc/tips/peer_to_peer_network_with_tor.mdwn
@@ -33,23 +33,38 @@ In both repositories, run this command:
 
 	git annex p2p --pair
 
-This will print out a code phrase, like "11-incredible-tumeric",
-and prompt for you to enter the other repository's code phrase.
+This will print out a pairing code, like "11-incredible-tumeric",
+and prompt for you to enter the other repository's pairing code.
 
-Once the code phrases are exchanged, the two repositories will be securely
+Once the pairing codes are exchanged, the two repositories will be securely
 connected to one-another via Tor. Each will have a git remote, with a name
 like "peer1", which connects to the other repository.
 
 Then, you can run commands like `git annex sync peer1 --content` to sync
 with the paired repository.
 
-The Magic Wormhole code phrases used during pairing will no longer be
-useful for anything afterwards.
-
 Pairing connects just two repositories, but you can repeat the process to
 pair with as many other repositories as you like, in order to build up
 larger networks of repositories.
 
+## how to exchange pairing codes
+
+When pairing with a friend's repository, you have to exchange
+pairing codes. How to do this securely?
+
+The pairing codes can only be used once, so it's ok to exchange them in
+a way that someone else can access later. However, if someone can overhear
+your exchange of codes in real time, they could trick you into pairing
+with them.
+
+Here are some suggestions for how to exchange the codes,
+with the most secure ways first:
+
+* In person.
+* In an encrypted message (gpg signed email, Off The Record (OTR)
+  conversation, etc).
+* By a voice phone call.
+
 ## starting git-annex remotedaemon on boot
 
 Notice the `git annex remotedaemon` being run in the above examples.

From a13b789d045428f9abaa901b3a0b311f1b539fa3 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 19 Dec 2016 15:54:47 -0400
Subject: [PATCH 330/367] followup

---
 ...omment_10_b21a337256c58953e1440317c0c1db80._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_10_b21a337256c58953e1440317c0c1db80._comment

diff --git a/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_10_b21a337256c58953e1440317c0c1db80._comment b/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_10_b21a337256c58953e1440317c0c1db80._comment
new file mode 100644
index 0000000000..15dde50f88
--- /dev/null
+++ b/doc/bugs/unable_to_decommit_memory__58___Invalid_argument/comment_10_b21a337256c58953e1440317c0c1db80._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 10"""
+ date="2016-12-19T19:53:11Z"
+ content="""
+The only way the files could be in lost+found is if the system crashed or
+there was a disk error etc. Can't be due to this bug. So, it may be that
+this bug did not actually cause any data loss. The screwed up symlinks could
+have been caused by a disk error.
+"""]]

From df5a0059ca85916c19c90d9525957f6e2dbca1db Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 19 Dec 2016 16:28:12 -0400
Subject: [PATCH 331/367] analysis

---
 Command/ImportFeed.hs                            | 10 ++++++----
 ...t_1_3c6a60ab9c772b95ca5205199554b914._comment | 16 ++++++++++++++++
 2 files changed, 22 insertions(+), 4 deletions(-)
 create mode 100644 doc/bugs/YouTube_-_error_in_importfeed/comment_1_3c6a60ab9c772b95ca5205199554b914._comment

diff --git a/Command/ImportFeed.hs b/Command/ImportFeed.hs
index 1736f2567c..832ec1246f 100644
--- a/Command/ImportFeed.hs
+++ b/Command/ImportFeed.hs
@@ -138,10 +138,12 @@ findDownloads u = go =<< downloadFeed u
 			Just $ ToDownload f u i $ Enclosure enclosureurl
 		Nothing -> mkquvi f i
 	mkquvi f i = case getItemLink i of
-		Just link -> ifM (quviSupported link)
-			( return $ Just $ ToDownload f u i $ QuviLink link
-			, return Nothing
-			)
+		Just link -> do
+			liftIO $ print ("link", link)
+			ifM (quviSupported link)
+				( return $ Just $ ToDownload f u i $ QuviLink link
+				, return Nothing
+				)
 		Nothing -> return Nothing
 
 {- Feeds change, so a feed download cannot be resumed. -}
diff --git a/doc/bugs/YouTube_-_error_in_importfeed/comment_1_3c6a60ab9c772b95ca5205199554b914._comment b/doc/bugs/YouTube_-_error_in_importfeed/comment_1_3c6a60ab9c772b95ca5205199554b914._comment
new file mode 100644
index 0000000000..afdff4942c
--- /dev/null
+++ b/doc/bugs/YouTube_-_error_in_importfeed/comment_1_3c6a60ab9c772b95ca5205199554b914._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-12-19T19:55:23Z"
+ content="""
+It's somewhat misleading that it complains there are no enclosures in the
+feed. While importfeed mostly downloads only enclosures in podcast feeds,
+it also checks link tags, which this feed contains, to see if quvi supports
+downloading content from them. Quvi does support the links in this feed,
+so it should work despite there being no enclosures.
+
+I've reproduced it not working, and it seems that the problem is this is
+not quite a valid Atom feed, and the feed parsing library is failing to
+parse it. Perhaps that can be improved; I filed a bug here
+<https://github.com/bergmark/feed/issues/18>
+"""]]

From 95c8b37544c383983712c5c368dd803c51bf8eeb Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 19 Dec 2016 17:03:52 -0400
Subject: [PATCH 332/367] Linux standalone: Improve generation of locale
 definition files, supporting locales such as, en_GB.UTF-8.

---
 CHANGELOG                                     |  2 ++
 ..._1765400777911cc61eb591b76c84ae89._comment | 21 +++++++++++++++++++
 standalone/linux/skel/runshell                |  2 +-
 3 files changed, 24 insertions(+), 1 deletion(-)
 create mode 100644 doc/bugs/Linux_standalone__39__s_metadata_--batch_can__39__t_parse_UTF-8/comment_1_1765400777911cc61eb591b76c84ae89._comment

diff --git a/CHANGELOG b/CHANGELOG
index 95d1355071..444d43b75f 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -17,6 +17,8 @@ git-annex (6.20161211) UNRELEASED; urgency=medium
   * p2p: --link no longer takes a remote name, instead the --name
     option can be used.
   * Debian: Build webapp on armel.
+  * Linux standalone: Improve generation of locale definition files,
+    supporting locales such as, en_GB.UTF-8.
 
  -- Joey Hess <id@joeyh.name>  Sun, 11 Dec 2016 21:29:51 -0400
 
diff --git a/doc/bugs/Linux_standalone__39__s_metadata_--batch_can__39__t_parse_UTF-8/comment_1_1765400777911cc61eb591b76c84ae89._comment b/doc/bugs/Linux_standalone__39__s_metadata_--batch_can__39__t_parse_UTF-8/comment_1_1765400777911cc61eb591b76c84ae89._comment
new file mode 100644
index 0000000000..cf591d65ed
--- /dev/null
+++ b/doc/bugs/Linux_standalone__39__s_metadata_--batch_can__39__t_parse_UTF-8/comment_1_1765400777911cc61eb591b76c84ae89._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-12-19T20:37:56Z"
+ content="""
+JSON uses  a UTF-8 encoding. So the usual hack used in git-annex
+of bypassing the system locale and essentially reading data as binary can't
+work for --json. 
+
+So, I think you need to be using a unicode locale, which is properly set up
+in order to use --json. And, the data fed in via --json needs to actually
+be encoded as unicode and not some other encoding.
+
+runshell was recently changed to bypass using the system locales, it
+includes its own locale data and attempts to generate a locale definition
+file for the locale. The code that did that was failing to notice that
+en_GB.UTF-8 was a UTF-8 locale (en_GB.utf8 would work though), which
+explains why the locale is not set inside runshell
+(git-annex.linux/git-annex is a script that uses runshell). I've corrected
+that problem, and verified it fixes the problem you reported.
+"""]]
diff --git a/standalone/linux/skel/runshell b/standalone/linux/skel/runshell
index d6bec7aa38..a4578eb1b2 100755
--- a/standalone/linux/skel/runshell
+++ b/standalone/linux/skel/runshell
@@ -132,7 +132,7 @@ for localeenv in "$LANG" "$LANGUAGE" "$LC_CTYPE" "$LC_NUMERIC" "$LC_TIME" \
 	if [ "$localeenv" != "$lastlocaleenv" ]; then
 		lastlocaleenv="$localeenv"
 		if [ ! -d "$base/locales/$localeenv" ]; then
-			if [ "${localeenv##[!.]*.}" = "utf8" ]; then
+			if [ "${localeenv##[!.]*.}" = "utf8" ] || [ "${localeenv##[!.]*.}" = "UTF-8" ]; then
 				(
 					rm -rf "$base/locales/$localeenv.new.$$" &&
 					mkdir -p "$base/locales/$localeenv.new.$$" &&

From 8136c72fa1af20ee00da072d48ac61f1143508b2 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 19 Dec 2016 18:08:27 -0400
Subject: [PATCH 333/367] further analysis

---
 ..._1765400777911cc61eb591b76c84ae89._comment | 40 +++++++++++++++----
 1 file changed, 32 insertions(+), 8 deletions(-)

diff --git a/doc/bugs/Linux_standalone__39__s_metadata_--batch_can__39__t_parse_UTF-8/comment_1_1765400777911cc61eb591b76c84ae89._comment b/doc/bugs/Linux_standalone__39__s_metadata_--batch_can__39__t_parse_UTF-8/comment_1_1765400777911cc61eb591b76c84ae89._comment
index cf591d65ed..4a15b19871 100644
--- a/doc/bugs/Linux_standalone__39__s_metadata_--batch_can__39__t_parse_UTF-8/comment_1_1765400777911cc61eb591b76c84ae89._comment
+++ b/doc/bugs/Linux_standalone__39__s_metadata_--batch_can__39__t_parse_UTF-8/comment_1_1765400777911cc61eb591b76c84ae89._comment
@@ -3,14 +3,6 @@
  subject="""comment 1"""
  date="2016-12-19T20:37:56Z"
  content="""
-JSON uses  a UTF-8 encoding. So the usual hack used in git-annex
-of bypassing the system locale and essentially reading data as binary can't
-work for --json. 
-
-So, I think you need to be using a unicode locale, which is properly set up
-in order to use --json. And, the data fed in via --json needs to actually
-be encoded as unicode and not some other encoding.
-
 runshell was recently changed to bypass using the system locales, it
 includes its own locale data and attempts to generate a locale definition
 file for the locale. The code that did that was failing to notice that
@@ -18,4 +10,36 @@ en_GB.UTF-8 was a UTF-8 locale (en_GB.utf8 would work though), which
 explains why the locale is not set inside runshell
 (git-annex.linux/git-annex is a script that uses runshell). I've corrected
 that problem, and verified it fixes the problem you reported.
+
+----
+
+However.. The same thing happens when using LANG=C with git-annex
+installed by any method and --json --batch. So the deeper problem is that
+it's forcing the batch input to be decoded as utf8 via the current locale.
+This happens in Command/MetaData.hs parseJSONInput which uses
+`BU.fromString`. 
+
+I tried swapping in `encodeBS` for `BU.fromString`. That prevented the
+decoding error, but made git-annex complain that the file was not annexed,
+due to a Mojibake problem: 
+
+With `encodeBS`, the input `{"file":"ü.txt"}` is encoded as
+`"{\"file\":\"\195\188.txt\"}"`. Aeson parses that input to this:
+
+	JSONActionItem {itemCommand = Nothing, itemKey = Nothing, itemFile = Just "\252.txt", itemAdded = Nothing}
+
+Note that the first two bytes have been
+parsed by Aeson as unicode (since JSON is unicode encoded),
+yielding character 252 (ü).
+
+In a unicode locale, this works ok, because the encoding layer is able to
+convert that unicode character back to two bytes 195 188 
+and finds the file on disk. But in a non-unicode locale, it doesn't know
+what to do with the unicode character, and in fact it gets discarded
+and so it looks for a file named ".txt".
+
+So, to make --batch --json input work in non-unicode locales, it would
+need, after parsing the json, to re-encode filenames (and perhaps other
+data), from utf8 to the filesystem encoding. I have not yet worked out how
+to do that.
 """]]

From a171e576b2f9658cc99facee55fc04ac1fb2c57b Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Mon, 19 Dec 2016 18:18:57 -0400
Subject: [PATCH 334/367] rekey --force: Incorrectly marked the new key's
 content as being present in the local repo even when it was not.

---
 CHANGELOG        | 2 ++
 Command/ReKey.hs | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index 444d43b75f..10b6a75770 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -19,6 +19,8 @@ git-annex (6.20161211) UNRELEASED; urgency=medium
   * Debian: Build webapp on armel.
   * Linux standalone: Improve generation of locale definition files,
     supporting locales such as, en_GB.UTF-8.
+  * rekey --force: Incorrectly marked the new key's content as being
+    present in the local repo even when it was not.
 
  -- Joey Hess <id@joeyh.name>  Sun, 11 Dec 2016 21:29:51 -0400
 
diff --git a/Command/ReKey.hs b/Command/ReKey.hs
index 4ddbd68b61..aaaaf7e37a 100644
--- a/Command/ReKey.hs
+++ b/Command/ReKey.hs
@@ -126,6 +126,6 @@ cleanup file oldkey newkey = do
 			Database.Keys.removeAssociatedFile oldkey 
 				=<< inRepo (toTopFilePath file)
 		)
-
-	logStatus newkey InfoPresent
+	whenM (inAnnex newkey) $
+		logStatus newkey InfoPresent
 	return True

From a58c74b502a8e8d7b92a31b2bbb07f356b3578e8 Mon Sep 17 00:00:00 2001
From: "m8r-achx62@7323980ed426b7f78c85dfefe7358672bce44e98" <m8r-achx62@web>
Date: Mon, 19 Dec 2016 22:33:14 +0000
Subject: [PATCH 335/367] Added a comment

---
 .../comment_2_fe28e0f76dbefb1963820011fc8fc3e7._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/bugs/YouTube_-_error_in_importfeed/comment_2_fe28e0f76dbefb1963820011fc8fc3e7._comment

diff --git a/doc/bugs/YouTube_-_error_in_importfeed/comment_2_fe28e0f76dbefb1963820011fc8fc3e7._comment b/doc/bugs/YouTube_-_error_in_importfeed/comment_2_fe28e0f76dbefb1963820011fc8fc3e7._comment
new file mode 100644
index 0000000000..edf4c855cd
--- /dev/null
+++ b/doc/bugs/YouTube_-_error_in_importfeed/comment_2_fe28e0f76dbefb1963820011fc8fc3e7._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="m8r-achx62@7323980ed426b7f78c85dfefe7358672bce44e98"
+ nickname="m8r-achx62"
+ avatar="http://cdn.libravatar.org/avatar/adaf4c4277529e10e32c467fa4ed4b9a"
+ subject="comment 2"
+ date="2016-12-19T22:33:13Z"
+ content="""
+Thanks for following this up Joey!
+"""]]

From 00179782ab36bb9701da81e394dd557679b5a0eb Mon Sep 17 00:00:00 2001
From: "veron_veron@8e19f168a8da3dabcdbf28ccd3f27edfb40941ed" <veron_veron@web>
Date: Tue, 20 Dec 2016 09:19:31 +0000
Subject: [PATCH 336/367] Added a comment: easy

---
 ...comment_5_29ec08578bc45e4bbdecf76d1eb33826._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/recover_deleted_files___63__/comment_5_29ec08578bc45e4bbdecf76d1eb33826._comment

diff --git a/doc/forum/recover_deleted_files___63__/comment_5_29ec08578bc45e4bbdecf76d1eb33826._comment b/doc/forum/recover_deleted_files___63__/comment_5_29ec08578bc45e4bbdecf76d1eb33826._comment
new file mode 100644
index 0000000000..a59b079643
--- /dev/null
+++ b/doc/forum/recover_deleted_files___63__/comment_5_29ec08578bc45e4bbdecf76d1eb33826._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="veron_veron@8e19f168a8da3dabcdbf28ccd3f27edfb40941ed"
+ nickname="veron_veron"
+ avatar="http://cdn.libravatar.org/avatar/eb7805af696010b4e8844aefeeb89a1b"
+ subject="easy"
+ date="2016-12-20T09:19:31Z"
+ content="""
+Indeed the recycle bin, and using Windows Explorer to look in folders; the programs you used are not designed to search for files; especially programs like Elements, which uses a catalog is not what you need now. Moving files confuses PS Elements organiser enough to make it seem like you lost files, while they're actually there. When things go wrong, use the basic tools provided in Windows itself (explorer, search).
+For file recovery programs, I've use <a href=\"http://www.passwordmanagers.net/resources/Software-for-Recovering-Deleted-Files-71.html\">Software for Recovering Deleted Files</a> on a few occassions; it's free and it works as good as can be expected. The main thing is to stop using the external hard disk until you use this tool, and avoid writing any files to it.
+"""]]

From 31347df435b20876ff6c46cf03dce46072e5f611 Mon Sep 17 00:00:00 2001
From: ilovezfs <ilovezfs@web>
Date: Tue, 20 Dec 2016 18:28:25 +0000
Subject: [PATCH 337/367]

---
 ...tility.Directory_and_Utility.FileSize.mdwn | 33 +++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 doc/bugs/getFileSize_conflict_between_Utility.Directory_and_Utility.FileSize.mdwn

diff --git a/doc/bugs/getFileSize_conflict_between_Utility.Directory_and_Utility.FileSize.mdwn b/doc/bugs/getFileSize_conflict_between_Utility.Directory_and_Utility.FileSize.mdwn
new file mode 100644
index 0000000000..b744f0049a
--- /dev/null
+++ b/doc/bugs/getFileSize_conflict_between_Utility.Directory_and_Utility.FileSize.mdwn
@@ -0,0 +1,33 @@
+### Please describe the problem.
+directory 1.3.0.0 causes a conflict for "getFileSize"
+
+### What steps will reproduce the problem?
+Build git-annex with directory 1.3.0.0 (first need to bump max directory version on concurrent-output (and aws if building with s3))
+
+### What version of git-annex are you using? On what operating system?
+6.20161210 on macOS 10.11 El Capitan
+
+
+### Please provide any additional information below.
+
+[[!format sh """
+[23 of 34] Compiling Common           ( Common.hs, dist/setup/Common.o )
+
+Common.hs:3:16: error:
+    Conflicting exports for ‘getFileSize’:
+       ‘module X’ exports ‘X.getFileSize’
+         imported from ‘Utility.Directory’ at Common.hs:28:1-29
+         (and originally defined in ‘System.Directory’)
+       ‘module X’ exports ‘X.getFileSize’
+         imported from ‘Utility.FileSize’ at Common.hs:34:1-28
+         (and originally defined at Utility/FileSize.hs:26:1-11)
+"""]]
+
+A fix, though possibly not best, is to make this change in Common.hs:
+[[!format sh """
+import Utility.Directory as X hiding (getFileSize)
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+Yes :)
+

From c3d94abd0a4db508ef48f7428700d78d9f51e52b Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 20 Dec 2016 15:01:05 -0400
Subject: [PATCH 338/367] reflow

---
 doc/workflow.mdwn | 89 ++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 80 insertions(+), 9 deletions(-)

diff --git a/doc/workflow.mdwn b/doc/workflow.mdwn
index ad11ec1375..fc9db9d9ad 100644
--- a/doc/workflow.mdwn
+++ b/doc/workflow.mdwn
@@ -1,26 +1,97 @@
-Git-annex supports a wide variety of workflows, a spectrum that ranges from completely automatic behavior where git-annex handles everything, through manual behavior where git-annex does only what you say when you tell it to, all the way down to internal behavior, where you have complete control and understand how everything is stored and exactly what changes are happening.
+Git-annex supports a wide variety of workflows, a spectrum that ranges from
+completely automatic behavior where git-annex handles everything, through
+manual behavior where git-annex does only what you say when you tell it to,
+all the way down to internal behavior, where you have complete control and
+understand how everything is stored and exactly what changes are happening.
 
-I will proceed to summarize all of these. I will begin at the automatic end, hoping that this is most useful, and drill down to the low level approaches. Note, however, that this is the opposite order of how git-annex was apparently developed. A list of workflows that started from manual, commandline usage would be much more intuitive, but you'd have to be willing to read the man page and wiki pages to get started, and that's pretty much what's already out there anyway.
+I will proceed to summarize all of these. I will begin at the automatic
+end, hoping that this is most useful, and drill down to the low level
+approaches. Note, however, that this is the opposite order of how git-annex
+was apparently developed. A list of workflows that started from manual,
+commandline usage would be much more intuitive, but you'd have to be
+willing to read the man page and wiki pages to get started, and that's
+pretty much what's already out there anyway.
 
 Note that for each of these levels of interaction, all the levels following will also work as well. So you can actually manually move annexed files around while the webapp is running, etc.
 
 # 1. [[git annex webapp|git-annex-webapp]]
-The [[`git annex webapp`|git-annex-webapp]] command launches a local web server which serves a graphical user interface and automatically manages git annex. It will attempt to guide you through the whole process and do everything for you. I think the intent is that no other commands are needed. This should be run on every machine that may produce file changes.
+
+The [[`git annex webapp`|git-annex-webapp]] command launches a local web
+server which serves a graphical user interface and automatically manages
+git annex. It will attempt to guide you through the whole process and do
+everything for you. I think the intent is that no other commands are
+needed. This should be run on every machine that may produce file changes.
 
 # 2. [[git annex assistant|git-annex-assistant]] without the webapp
-You could call [[`git annex assistant`|git-annex-assistant]] the command-line version of the webapp, giving you more control over creating and connecting your repositories, and configuring how files are moved between them. The assistant, when running, will automatically watch for file changes and synchronize them to other repositories, but you must manually create the repositories and configure the rules for syncing. To create a repository, use `git init` and then [[`git annex init`|git-annex-init]], and then `git remote add` it to any other repositories. If you want more than one annex, you can add their paths to `~/.config/git-annex/autostart` if you would like them to automatically begin syncing when `git annex assistant --autostart` is run, perhaps on boot or login. You can configure rules for where files are copied using the repository setup commands such as [[git annex preferred-content|git-annex-preferred-content]] to configure [[content preferences|preferred content]] for what goes where, [[`git annex numcopies`|git-annex-numcopies]] for how many [[copies]] must be kept of each file, and [[`git config annex.largefiles`|tips/largefiles]] to define small files that should be stored straight in git; most of the settings are accessible in one place with [[`git annex vicfg`|git-annex-vicfg]].
+
+You could call [[`git annex assistant`|git-annex-assistant]] the
+command-line version of the webapp, giving you more control over creating
+and connecting your repositories, and configuring how files are moved
+between them. The assistant, when running, will automatically watch for
+file changes and synchronize them to other repositories, but you must
+manually create the repositories and configure the rules for syncing. To
+create a repository, use `git init` and then [[`git annex
+init`|git-annex-init]], and then `git remote add` it to any other
+repositories. If you want more than one annex, you can add their paths to
+`~/.config/git-annex/autostart` if you would like them to automatically
+begin syncing when `git annex assistant --autostart` is run, perhaps on
+boot or login. You can configure rules for where files are copied using the
+repository setup commands such as [[git annex
+preferred-content|git-annex-preferred-content]] to configure [[content
+preferences|preferred content]] for what goes where, [[`git annex
+numcopies`|git-annex-numcopies]] for how many [[copies]] must be kept of
+each file, and [[`git config annex.largefiles`|tips/largefiles]] to define
+small files that should be stored straight in git; most of the settings are
+accessible in one place with [[`git annex vicfg`|git-annex-vicfg]].
 
 # 3. [[git annex watch|git-annex-watch]] without the assistant
-The [[`git annex watch`|git-annex-watch]] command is like the assistant but has no automatic network behavior, giving you complete control over when repositories are pushed and pulled, and when files are moved between systems. The local repository is watched, and any file changes are added to git-annex. In order to synchronize between repositories, you must run [[`git annex sync --content`|git-annex-sync]] in the repository with the changes, which will merge the git history and logs with your remotes, and automatically transfer files to match your preferred and required content expressions.
+
+The [[`git annex watch`|git-annex-watch]] command is like the assistant but
+has no automatic network behavior, giving you complete control over when
+repositories are pushed and pulled, and when files are moved between
+systems. The local repository is watched, and any file changes are added to
+git-annex. In order to synchronize between repositories, you must run
+[[`git annex sync --content`|git-annex-sync]] in the repository with the
+changes, which will merge the git history and logs with your remotes, and
+automatically transfer files to match your preferred and required content
+expressions.
 
 # 4. No background processes
-This allows you to decide when and what files are annexed. In order to tell git-annex to manage files, you must [[`git annex add`|git-annex-add]] the files.
+
+This allows you to decide when and what files are annexed. In order to tell
+git-annex to manage files, you must [[`git annex add`|git-annex-add]] the
+files.
 
 # 5. Plain [[git annex sync|git-annex-sync]] without `--content`
-This gives you fine-grained control of where copies of your files are stored. [[`git annex sync`|git-annex-sync]] without `--content` tells git-annex to merge git histories, but it does not automatically transfer your large files between systems. To transfer files and directories, you can use [[`git annex get`|git-annex-get]], [[`git annex drop`|git-annex-drop]], [[`git annex move`|git-annex-move]], and [[`git annex copy`|git-annex-copy]]. Git-annex will not violate a required content expression or your numcopies setting unless you pass `--force`, so your files are still safe. This is the workflow I mostly use, and I find it the most stable. I'm trying to migrate up to `--content`, but I have too many large files that haven't reached their numcopies yet for that to be effective.
+
+This gives you fine-grained control of where copies of your files are
+stored. [[`git annex sync`|git-annex-sync]] without `--content` tells
+git-annex to merge git histories, but it does not automatically transfer
+your large files between systems. To transfer files and directories, you
+can use [[`git annex get`|git-annex-get]], [[`git annex
+drop`|git-annex-drop]], [[`git annex move`|git-annex-move]], and [[`git
+annex copy`|git-annex-copy]]. Git-annex will not violate a required content
+expression or your numcopies setting unless you pass `--force`, so your
+files are still safe. This is the workflow I mostly use, and I find it the
+most stable. I'm trying to migrate up to `--content`, but I have too many
+large files that haven't reached their numcopies yet for that to be
+effective.
 
 # 6. Manual management of git history without the synchronizer
-This allows you to control precisely what is committed to git, what commit message is used, and how your history is merged between repositories. You must have an understanding of git, and run `git commit` after `git annex add` to store the change. You must manage the git history yourself, using `git pull` and `git push`, to synchronize repositories. You may freely use git normally side-by-side with git-annex.
+
+This allows you to control precisely what is committed to git, what commit
+message is used, and how your history is merged between repositories. You
+must have an understanding of git, and run `git commit` after `git annex
+add` to store the change. You must manage the git history yourself, using
+`git pull` and `git push`, to synchronize repositories. You may freely use
+git normally side-by-side with git-annex.
 
 # 7. Manual management of git annex keys
-This gives you control of what and where git annex stores your files under the hood, and how they are associated with your working tree, rather than using the `git annex add` and `git annex get` commands which reference files automatically. Git-annex has a variety of plumbing commands listed in the [[man page|git-annex]] that let you directly store and retrieve data in an annex associated with your git repository, where every datafile is enumerated by a unique hashkey.
+
+This gives you control of what and where git annex stores your files under
+the hood, and how they are associated with your working tree, rather than
+using the `git annex add` and `git annex get` commands which reference
+files automatically. Git-annex has a variety of plumbing commands listed in
+the [[man page|git-annex]] that let you directly store and retrieve data in
+an annex associated with your git repository, where every datafile is
+enumerated by a unique hashkey.

From 116e601989a18a3f9931651e61cd934fa475560f Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 20 Dec 2016 15:03:28 -0400
Subject: [PATCH 339/367] firm up some language and remove a bit

---
 doc/workflow.mdwn | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/doc/workflow.mdwn b/doc/workflow.mdwn
index fc9db9d9ad..52f11e217f 100644
--- a/doc/workflow.mdwn
+++ b/doc/workflow.mdwn
@@ -7,7 +7,7 @@ understand how everything is stored and exactly what changes are happening.
 I will proceed to summarize all of these. I will begin at the automatic
 end, hoping that this is most useful, and drill down to the low level
 approaches. Note, however, that this is the opposite order of how git-annex
-was apparently developed. A list of workflows that started from manual,
+was developed. A list of workflows that started from manual,
 commandline usage would be much more intuitive, but you'd have to be
 willing to read the man page and wiki pages to get started, and that's
 pretty much what's already out there anyway.
@@ -19,7 +19,7 @@ Note that for each of these levels of interaction, all the levels following will
 The [[`git annex webapp`|git-annex-webapp]] command launches a local web
 server which serves a graphical user interface and automatically manages
 git annex. It will attempt to guide you through the whole process and do
-everything for you. I think the intent is that no other commands are
+everything for you. The intent is that no other commands are
 needed. This should be run on every machine that may produce file changes.
 
 # 2. [[git annex assistant|git-annex-assistant]] without the webapp
@@ -72,10 +72,7 @@ can use [[`git annex get`|git-annex-get]], [[`git annex
 drop`|git-annex-drop]], [[`git annex move`|git-annex-move]], and [[`git
 annex copy`|git-annex-copy]]. Git-annex will not violate a required content
 expression or your numcopies setting unless you pass `--force`, so your
-files are still safe. This is the workflow I mostly use, and I find it the
-most stable. I'm trying to migrate up to `--content`, but I have too many
-large files that haven't reached their numcopies yet for that to be
-effective.
+files are still safe.
 
 # 6. Manual management of git history without the synchronizer
 

From 9bdb617484b77607fb483ebde5a423f70ab67466 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 20 Dec 2016 15:10:33 -0400
Subject: [PATCH 340/367] add some links to new workflow page

---
 doc/how_it_works.mdwn                         |  7 +++--
 doc/links/key_concepts.mdwn                   |  1 +
 doc/sync.mdwn                                 |  2 ++
 ..._6ec6fb45021ba82ed6a4bb9a6f3cfceb._comment |  9 ++++++
 doc/walkthrough.mdwn                          |  7 ++++-
 doc/workflow.mdwn                             | 29 ++++++++++---------
 6 files changed, 39 insertions(+), 16 deletions(-)
 create mode 100644 doc/todo/Workflow_guide/comment_5_6ec6fb45021ba82ed6a4bb9a6f3cfceb._comment

diff --git a/doc/how_it_works.mdwn b/doc/how_it_works.mdwn
index 69e5256e3a..21fa39ea7e 100644
--- a/doc/how_it_works.mdwn
+++ b/doc/how_it_works.mdwn
@@ -1,8 +1,11 @@
-This page gives a high-level view of git-annex. For a detailed
+This page gives a high-level view of how git-annex works. For a detailed
 low-level view, see [[the_man_page|git-annex]] and [[internals]].
 
 You do not need to read this page to get started with using git-annex. The
-[[walkthrough]] provides step-by-step instructions.
+[[walkthrough]] provides step-by-step examples, and [[workflow]] discusses
+different ways you can use git-annex.
+
+----
 
 Still reading? Ok. Git's man page calls it "a stupid content
 tracker". With git-annex, git is instead "a stupid filename and metadata"
diff --git a/doc/links/key_concepts.mdwn b/doc/links/key_concepts.mdwn
index f1754e0c83..3dc2f6c5d6 100644
--- a/doc/links/key_concepts.mdwn
+++ b/doc/links/key_concepts.mdwn
@@ -3,5 +3,6 @@
 * [[git-annex man page|git-annex]]
 * [[how_it_works]]
 * [[special_remotes]]
+* [[workflows|workflow]]
 * [[sync]]
 * [[direct_mode]]
diff --git a/doc/sync.mdwn b/doc/sync.mdwn
index 0250d2fef0..cddccd1124 100644
--- a/doc/sync.mdwn
+++ b/doc/sync.mdwn
@@ -42,3 +42,5 @@ repositories, but does not transfer the content of annexed files. If you
 want to fully synchronise two repositories content,
 you can use `git annex sync --content`. You can also configure
 [[preferred_content]] settings to make only some content be synced.
+
+See [[git-annex-sync]] for the command's man page.
diff --git a/doc/todo/Workflow_guide/comment_5_6ec6fb45021ba82ed6a4bb9a6f3cfceb._comment b/doc/todo/Workflow_guide/comment_5_6ec6fb45021ba82ed6a4bb9a6f3cfceb._comment
new file mode 100644
index 0000000000..8b5b37ea35
--- /dev/null
+++ b/doc/todo/Workflow_guide/comment_5_6ec6fb45021ba82ed6a4bb9a6f3cfceb._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2016-12-20T19:04:12Z"
+ content="""
+Good start on the workflow page!
+
+I've added some links to it to make it discoverable.
+"""]]
diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn
index 22c94d570f..b35cf808d8 100644
--- a/doc/walkthrough.mdwn
+++ b/doc/walkthrough.mdwn
@@ -1,4 +1,9 @@
-A walkthrough of the basic features of git-annex.
+A walkthrough of some of the basic features of git-annex, using the command
+line. If you don't want to use the command line, see [[assistant/quickstart]]
+instead.
+
+What follows is only one possible [[workflow]] for using git-annex,
+but following along will teach you the basic concepts from the ground up.
 
 [[!toc]]
 
diff --git a/doc/workflow.mdwn b/doc/workflow.mdwn
index 52f11e217f..042b4bab43 100644
--- a/doc/workflow.mdwn
+++ b/doc/workflow.mdwn
@@ -12,7 +12,9 @@ commandline usage would be much more intuitive, but you'd have to be
 willing to read the man page and wiki pages to get started, and that's
 pretty much what's already out there anyway.
 
-Note that for each of these levels of interaction, all the levels following will also work as well. So you can actually manually move annexed files around while the webapp is running, etc.
+Note that for each of these levels of interaction, all the levels following
+will also work as well. So you can actually manually move annexed files
+around while the webapp is running, etc.
 
 # 1. [[git annex webapp|git-annex-webapp]]
 
@@ -27,18 +29,19 @@ needed. This should be run on every machine that may produce file changes.
 You could call [[`git annex assistant`|git-annex-assistant]] the
 command-line version of the webapp, giving you more control over creating
 and connecting your repositories, and configuring how files are moved
-between them. The assistant, when running, will automatically watch for
-file changes and synchronize them to other repositories, but you must
-manually create the repositories and configure the rules for syncing. To
-create a repository, use `git init` and then [[`git annex
-init`|git-annex-init]], and then `git remote add` it to any other
-repositories. If you want more than one annex, you can add their paths to
-`~/.config/git-annex/autostart` if you would like them to automatically
-begin syncing when `git annex assistant --autostart` is run, perhaps on
-boot or login. You can configure rules for where files are copied using the
-repository setup commands such as [[git annex
-preferred-content|git-annex-preferred-content]] to configure [[content
-preferences|preferred content]] for what goes where, [[`git annex
+between them.
+
+The assistant, when running, will automatically watch for file changes and
+synchronize them to other repositories, but you must manually create the
+repositories and configure the rules for syncing. To create a repository,
+use `git init` and then [[`git annex init`|git-annex-init]], and then `git
+remote add` it to any other repositories. If you want more than one annex,
+you can add their paths to `~/.config/git-annex/autostart` if you would
+like them to automatically begin syncing when `git annex assistant
+--autostart` is run, perhaps on boot or login. You can configure rules for
+where files are copied using the repository setup commands such as [[git
+annex preferred-content|git-annex-preferred-content]] to configure
+[[content preferences|preferred content]] for what goes where, [[`git annex
 numcopies`|git-annex-numcopies]] for how many [[copies]] must be kept of
 each file, and [[`git config annex.largefiles`|tips/largefiles]] to define
 small files that should be stored straight in git; most of the settings are

From f3972999ae7d427428182e46abc3c60c5aae959b Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 20 Dec 2016 15:15:01 -0400
Subject: [PATCH 341/367] update

---
 ...comment_5_6ec6fb45021ba82ed6a4bb9a6f3cfceb._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/doc/todo/Workflow_guide/comment_5_6ec6fb45021ba82ed6a4bb9a6f3cfceb._comment b/doc/todo/Workflow_guide/comment_5_6ec6fb45021ba82ed6a4bb9a6f3cfceb._comment
index 8b5b37ea35..fe30f61064 100644
--- a/doc/todo/Workflow_guide/comment_5_6ec6fb45021ba82ed6a4bb9a6f3cfceb._comment
+++ b/doc/todo/Workflow_guide/comment_5_6ec6fb45021ba82ed6a4bb9a6f3cfceb._comment
@@ -6,4 +6,14 @@
 Good start on the workflow page!
 
 I've added some links to it to make it discoverable.
+
+Not sure if the workflow page quite gets to what was originally requested:
+
+> I want to start keeping track of some files I have in a directory
+> I want to copy them to a second computer.
+> From a third place, I want to get them from the second computer.
+> I change the files on one computer, and I want to make sure the changes get synced to the others.
+> What are the commands you'd run at each step?
+
+Leaving this todo open for now..
 """]]

From e312ec37506f4b07beb0e082fedbdd06aed24c42 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 20 Dec 2016 15:23:59 -0400
Subject: [PATCH 342/367] Fix build with directory-1.3.

See https://github.com/haskell/directory/issues/66
---
 CHANGELOG                                                       | 1 +
 Utility/SystemDirectory.hs                                      | 2 +-
 ...conflict_between_Utility.Directory_and_Utility.FileSize.mdwn | 1 +
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG b/CHANGELOG
index 10b6a75770..04b0bd96a2 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -21,6 +21,7 @@ git-annex (6.20161211) UNRELEASED; urgency=medium
     supporting locales such as, en_GB.UTF-8.
   * rekey --force: Incorrectly marked the new key's content as being
     present in the local repo even when it was not.
+  * Fix build with directory-1.3.
 
  -- Joey Hess <id@joeyh.name>  Sun, 11 Dec 2016 21:29:51 -0400
 
diff --git a/Utility/SystemDirectory.hs b/Utility/SystemDirectory.hs
index 3dd44d1993..b9040fe13f 100644
--- a/Utility/SystemDirectory.hs
+++ b/Utility/SystemDirectory.hs
@@ -13,4 +13,4 @@ module Utility.SystemDirectory (
 	module System.Directory
 ) where
 
-import System.Directory hiding (isSymbolicLink)
+import System.Directory hiding (isSymbolicLink, getFileSize)
diff --git a/doc/bugs/getFileSize_conflict_between_Utility.Directory_and_Utility.FileSize.mdwn b/doc/bugs/getFileSize_conflict_between_Utility.Directory_and_Utility.FileSize.mdwn
index b744f0049a..75a5461599 100644
--- a/doc/bugs/getFileSize_conflict_between_Utility.Directory_and_Utility.FileSize.mdwn
+++ b/doc/bugs/getFileSize_conflict_between_Utility.Directory_and_Utility.FileSize.mdwn
@@ -31,3 +31,4 @@ import Utility.Directory as X hiding (getFileSize)
 ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
 Yes :)
 
+> [[fixed|done]]; thanks for reporting!

From 8f3b2c206cd65d5cc7afedee819e1317401b0609 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 20 Dec 2016 15:26:14 -0400
Subject: [PATCH 343/367] Debian: Suggest tor and magic-wormhole.

Suggests, not recommends, because tor is not for everyone.
---
 CHANGELOG      | 4 ++--
 debian/control | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index 04b0bd96a2..9d9faf1e4c 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -2,7 +2,6 @@ git-annex (6.20161211) UNRELEASED; urgency=medium
 
   * p2p --pair makes it easy to pair repositories over P2P, using
     Magic Wormhole codes to find the other repository.
-  * Debian: Recommend magic-wormhole.
   * metadata --batch: Fix bug when conflicting metadata changes were
     made in the same batch run.
   * Pass annex.web-options to wget and curl after other options, so that
@@ -16,12 +15,13 @@ git-annex (6.20161211) UNRELEASED; urgency=medium
     be processed without requiring it to be in the current encoding.
   * p2p: --link no longer takes a remote name, instead the --name
     option can be used.
-  * Debian: Build webapp on armel.
   * Linux standalone: Improve generation of locale definition files,
     supporting locales such as, en_GB.UTF-8.
   * rekey --force: Incorrectly marked the new key's content as being
     present in the local repo even when it was not.
   * Fix build with directory-1.3.
+  * Debian: Suggest tor and magic-wormhole.
+  * Debian: Build webapp on armel.
 
  -- Joey Hess <id@joeyh.name>  Sun, 11 Dec 2016 21:29:51 -0400
 
diff --git a/debian/control b/debian/control
index 644c220355..b64f3e3be9 100644
--- a/debian/control
+++ b/debian/control
@@ -112,10 +112,11 @@ Recommends:
 	git-remote-gcrypt (>= 0.20130908-6),
 	nocache,
 	aria2,
-	magic-wormhole,
 Suggests:
 	xdot,
 	bup,
+	tor,
+	magic-wormhole,
 	tahoe-lafs,
 	libnss-mdns,
 Description: manage files with git, without checking their contents into git

From 477aa442096c8ced0a98e06e9da2f2f8d1acbc32 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 20 Dec 2016 15:38:18 -0400
Subject: [PATCH 344/367] anwser

---
 ..._17ab85276bcf495a656c7091753c086f._comment | 60 +++++++++++++++++++
 1 file changed, 60 insertions(+)
 create mode 100644 doc/forum/Git-annex_link_to_different_file_names/comment_1_17ab85276bcf495a656c7091753c086f._comment

diff --git a/doc/forum/Git-annex_link_to_different_file_names/comment_1_17ab85276bcf495a656c7091753c086f._comment b/doc/forum/Git-annex_link_to_different_file_names/comment_1_17ab85276bcf495a656c7091753c086f._comment
new file mode 100644
index 0000000000..8085253ad5
--- /dev/null
+++ b/doc/forum/Git-annex_link_to_different_file_names/comment_1_17ab85276bcf495a656c7091753c086f._comment
@@ -0,0 +1,60 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2016-12-20T19:27:31Z"
+ content="""
+Yes, you can do this. Effectively, you have two branches. In the master
+branch, you have drawing.pdf with a single name and changes commited to it.
+In the coworkers branch, you have the multiple different versions. Git has
+no difficulty representing this, but it's up to you to maintain the
+different branches.
+
+For example:
+
+	joey@darkstar:~/tmp/shop>git commit -m 'updated drawing some more' 
+	[master 1403dd4] updated drawing some more
+	 1 file changed, 1 insertion(+), 1 deletion(-)
+	joey@darkstar:~/tmp/shop>git checkout coworkers 
+	Switched to branch 'coworkers'
+	joey@darkstar:~/tmp/shop#coworkers>git show master
+	commit 1403dd49b2c378e78b8b8ec82d73e295c486697b
+	Author: Joey Hess <joeyh@joeyh.name>
+	Date:   Tue Dec 20 15:31:17 2016 -0400
+	
+	    updated drawing some more
+	
+	diff --git a/drawing.pdf b/drawing.pdf
+	index b59371e..c05ed95 120000
+	--- a/drawing.pdf
+	+++ b/drawing.pdf
+	@@ -1 +1 @@
+	-.git/annex/objects/55/MZ/SHA256E-s13--c5f6529491f9e6d40e893d2ffc008bc297bcc56a680040c124e4019fb5c1a94d.pdf/SHA256E-s13--c5f6529491f9e6d40e893d2ffc008bc297bcc56a680040c124e4019fb5c1a94d.pdf
+	\ No newline at end of file
+	+.git/annex/objects/xj/XF/SHA256E-s17--7786e857a89634ff9242d899245cbcc5e009736af6b0553cb7283b2daef77d16.pdf/SHA256E-s17--7786e857a89634ff9242d899245cbcc5e009736af6b0553cb7283b2daef77d16.pdf
+	\ No newline at end of file
+	joey@darkstar:~/tmp/shop#coworkers>ln -s .git/annex/objects/xj/XF/SHA256E-s17--7786e857a89634ff9242d899245cbcc5e009736af6b0553cb7283b2daef77d16.pdf/SHA256E-s17--7786e857a89634ff9242d899245cbcc5e009736af6b0553cb7283b2daef77d16.pdf drawing_rev2.pdf
+	joey@darkstar:~/tmp/shop#coworkers>git add drawing_rev2.pdf 
+	joey@darkstar:~/tmp/shop#coworkers>ls
+	drawing.pdf@  drawing_rev2.pdf@
+	joey@darkstar:~/tmp/shop#coworkers>git commit -m 'added rev2 of drawing'
+	[coworkers cf27781] added rev2 of drawing
+	 1 file changed, 1 insertion(+)
+	  create mode 120000 drawing_rev2.pdf
+
+In the example, I looked at what was committed to master, and copied and
+pasted the git-annex symlink into a new drawing_rev2.pdf file.
+
+That's the basic idea. There might be a better way to do that. Another way,
+for example, would be to have 2 clones of the repo, one with master checked
+out and one with coworkers checked out. You could then run, in the
+coworkers checkout:
+
+	cp -a ../master/drawing.pdf drawing_rev2.pdf
+	git add drawing_rev2.pdf
+	git commit -m 'added rev2 of drawing'
+
+That results in the same commit as the method I showed.
+
+With some scripting, you should be able to automate keeping the two
+branches in sync.
+"""]]

From 944a6503b9f6e5a1090d278748e5fa99efd45ad9 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 20 Dec 2016 16:01:10 -0400
Subject: [PATCH 345/367] relocate tor socket out of /etc

weasel explained that apparmor limits on what files tor can read do not
apply to sockets (because they're not files). And apparently the
problems I was seeing with hidden services not being accessible had to
do with onion address propigation and not the location of the socket
file.

remotedaemon looks up the HiddenServicePort in torrc, so if it was
previously configured with the socket in /etc, that will still work.

This commit was sponsored by Denis Dzyubenko on Patreon.
---
 CHANGELOG                     |  2 ++
 Command/EnableTor.hs          |  2 +-
 RemoteDaemon/Transport/Tor.hs | 61 +++++++++++++++++++----------------
 Utility/Tor.hs                | 54 ++++++++++++++++++++-----------
 4 files changed, 71 insertions(+), 48 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index 9d9faf1e4c..65a3da82b3 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -19,6 +19,8 @@ git-annex (6.20161211) UNRELEASED; urgency=medium
     supporting locales such as, en_GB.UTF-8.
   * rekey --force: Incorrectly marked the new key's content as being
     present in the local repo even when it was not.
+  * enable-tor: Put tor sockets in /var/lib/tor-annex/, rather
+    than in /etc/tor/hidden_service/.
   * Fix build with directory-1.3.
   * Debian: Suggest tor and magic-wormhole.
   * Debian: Build webapp on armel.
diff --git a/Command/EnableTor.hs b/Command/EnableTor.hs
index d24ecb2dc7..c6d477b4e7 100644
--- a/Command/EnableTor.hs
+++ b/Command/EnableTor.hs
@@ -30,6 +30,6 @@ start ps = case readish =<< headMaybe ps of
 		when (uuid == NoUUID) $
 			giveup "This can only be run in a git-annex repository."
 		(onionaddr, onionport) <- liftIO $
-			addHiddenService userid (fromUUID uuid)
+			addHiddenService "tor-annex" userid (fromUUID uuid)
 		storeP2PAddress $ TorAnnex onionaddr onionport
 		stop
diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs
index 61e1189a54..3f70fb1fbb 100644
--- a/RemoteDaemon/Transport/Tor.hs
+++ b/RemoteDaemon/Transport/Tor.hs
@@ -39,36 +39,41 @@ import qualified Network.Socket as S
 server :: TransportHandle -> IO ()
 server th@(TransportHandle (LocalRepo r) _) = do
 	u <- liftAnnex th getUUID
-
-	q <- newTBMQueueIO maxConnections
-	replicateM_ maxConnections $
-		forkIO $ forever $ serveClient th u r q
-
 	uid <- getRealUserID
 	let ident = fromUUID u
-	let sock = hiddenServiceSocketFile uid ident
-	nukeFile sock
-	soc <- S.socket S.AF_UNIX S.Stream S.defaultProtocol
-	S.bind soc (S.SockAddrUnix sock)
-	-- Allow everyone to read and write to the socket; tor is probably
-	-- running as a different user. Connections have to authenticate
-	-- to do anything, so it's fine that other local users can connect.
-	modifyFileMode sock $ addModes
-		[groupReadMode, groupWriteMode, otherReadMode, otherWriteMode]
-	S.listen soc 2
-	debugM "remotedaemon" "Tor hidden service running"
-	forever $ do
-		(conn, _) <- S.accept soc
-		h <- setupHandle conn
-		ok <- atomically $ ifM (isFullTBMQueue q)
-			( return False
-			, do
-				writeTBMQueue q h
-				return True
-			)
-		unless ok $ do
-			hClose h
-			warningIO "dropped Tor connection, too busy"
+	go u =<< getHiddenServiceSocketFile uid ident
+  where
+	go u (Just sock) = do
+		q <- newTBMQueueIO maxConnections
+		replicateM_ maxConnections $
+			forkIO $ forever $ serveClient th u r q
+
+		nukeFile sock
+		soc <- S.socket S.AF_UNIX S.Stream S.defaultProtocol
+		S.bind soc (S.SockAddrUnix sock)
+		-- Allow everyone to read and write to the socket; tor
+		-- is probably running as a different user.
+		-- Connections have to authenticate to do anything,
+		-- so it's fine that other local users can connect to the
+		-- socket.
+		modifyFileMode sock $ addModes
+			[groupReadMode, groupWriteMode, otherReadMode, otherWriteMode]
+
+		S.listen soc 2
+		debugM "remotedaemon" "Tor hidden service running"
+		forever $ do
+			(conn, _) <- S.accept soc
+			h <- setupHandle conn
+			ok <- atomically $ ifM (isFullTBMQueue q)
+				( return False
+				, do
+					writeTBMQueue q h
+					return True
+				)
+			unless ok $ do
+				hClose h
+				warningIO "dropped Tor connection, too busy"
+	go _ Nothing = debugM "remotedaemon" "Tor hidden service not enabled"
 
 -- How many clients to serve at a time, maximum. This is to avoid DOS attacks.
 maxConnections :: Int
diff --git a/Utility/Tor.hs b/Utility/Tor.hs
index 71628084ce..64a6ae11da 100644
--- a/Utility/Tor.hs
+++ b/Utility/Tor.hs
@@ -25,8 +25,12 @@ newtype OnionAddress = OnionAddress String
 
 type OnionSocket = FilePath
 
+-- | A unique identifier for a hidden service.
 type UniqueIdent = String
 
+-- | Name of application that is providing a hidden service.
+type AppName = String
+
 connectHiddenService :: OnionAddress -> OnionPort -> IO Socket
 connectHiddenService (OnionAddress address) port = do
 	(s, _) <- socksConnect torsockconf socksaddr
@@ -48,9 +52,9 @@ connectHiddenService (OnionAddress address) port = do
 -- 
 -- If there is already a hidden service for the specified unique
 -- identifier, returns its information without making any changes.
-addHiddenService :: UserID -> UniqueIdent -> IO (OnionAddress, OnionPort)
-addHiddenService uid ident = do
-	prepHiddenServiceSocketDir uid ident
+addHiddenService :: AppName -> UserID -> UniqueIdent -> IO (OnionAddress, OnionPort)
+addHiddenService appname uid ident = do
+	prepHiddenServiceSocketDir appname uid ident
 	ls <- lines <$> readFile torrc
 	let portssocks = mapMaybe (parseportsock . separate isSpace) ls
 	case filter (\(_, s) -> s == sockfile) portssocks of
@@ -81,7 +85,7 @@ addHiddenService uid ident = do
 		return (p, drop 1 (dropWhile (/= ':') l))
 	parseportsock _ = Nothing
 	
-	sockfile = hiddenServiceSocketFile uid ident
+	sockfile = hiddenServiceSocketFile appname uid ident
 
 	-- An infinite random list of high ports.
 	mkhighports g = 
@@ -104,7 +108,7 @@ addHiddenService uid ident = do
 -- The "hs" is used in the name to prevent too long a path name,
 -- which could present problems for socketFile.
 hiddenServiceDir :: UserID -> UniqueIdent -> FilePath
-hiddenServiceDir uid ident = libDir </> "hs_" ++ show uid ++ "_" ++ ident
+hiddenServiceDir uid ident = torLibDir </> "hs_" ++ show uid ++ "_" ++ ident
 
 hiddenServiceHostnameFile :: UserID -> UniqueIdent -> FilePath
 hiddenServiceHostnameFile uid ident = hiddenServiceDir uid ident </> "hostname"
@@ -112,33 +116,45 @@ hiddenServiceHostnameFile uid ident = hiddenServiceDir uid ident </> "hostname"
 -- | Location of the socket for a hidden service.
 --
 -- This has to be a location that tor can read from, and that the user
--- can write to. Tor is often prevented by apparmor from reading
--- from many locations. Putting it in /etc is a FHS violation, but it's the
--- simplest and most robust choice until http://bugs.debian.org/846275 is
--- dealt with.
+-- can write to. Since torLibDir is locked down, it can't go in there.
 --
 -- Note that some unix systems limit socket paths to 92 bytes long.
 -- That should not be a problem if the UniqueIdent is around the length of
--- a UUID.
-hiddenServiceSocketFile :: UserID -> UniqueIdent -> FilePath
-hiddenServiceSocketFile uid ident = etcDir </> "hidden_services" </> show uid ++ "_" ++ ident </> "s"
+-- a UUID, and the AppName is short.
+hiddenServiceSocketFile :: AppName -> UserID -> UniqueIdent -> FilePath
+hiddenServiceSocketFile appname uid ident = varLibDir </> appname </> show uid ++ "_" ++ ident </> "s"
+
+-- | Parse torrc, to get the socket file used for a hidden service with
+-- the specified UniqueIdent.
+getHiddenServiceSocketFile :: UserID -> UniqueIdent -> IO (Maybe FilePath)
+getHiddenServiceSocketFile uid ident = 
+	parse . map words . lines <$> catchDefaultIO "" (readFile torrc)
+  where
+	parse [] = Nothing
+	parse (("HiddenServiceDir":hsdir:[]):("HiddenServicePort":_hsport:hsaddr:[]):rest)
+		| "unix:" `isPrefixOf` hsaddr && hsdir == hsdir_want =
+			Just (drop (length "unix:") hsaddr)
+		| otherwise = parse rest
+	parse (_:rest) = parse rest
+
+	hsdir_want = hiddenServiceDir uid ident
 
 -- | Sets up the directory for the socketFile, with appropriate
 -- permissions. Must run as root.
-prepHiddenServiceSocketDir :: UserID -> UniqueIdent -> IO ()
-prepHiddenServiceSocketDir uid ident = do
+prepHiddenServiceSocketDir :: AppName -> UserID -> UniqueIdent -> IO ()
+prepHiddenServiceSocketDir appname uid ident = do
 	createDirectoryIfMissing True d
 	setOwnerAndGroup d uid (-1)
 	modifyFileMode d $
 		addModes [ownerReadMode, ownerExecuteMode, ownerWriteMode]
   where
-	d = takeDirectory $ hiddenServiceSocketFile uid ident
+	d = takeDirectory $ hiddenServiceSocketFile appname uid ident
 
 torrc :: FilePath
 torrc = "/etc/tor/torrc"
 
-libDir :: FilePath
-libDir = "/var/lib/tor"
+torLibDir :: FilePath
+torLibDir = "/var/lib/tor"
 
-etcDir :: FilePath
-etcDir = "/etc/tor"
+varLibDir :: FilePath
+varLibDir = "/var/lib"

From f7ca2b92fb76ae9ed3d1be82a4b403ec39dff9d9 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 20 Dec 2016 17:40:36 -0400
Subject: [PATCH 346/367] enable-tor: No longer needs to be run as root.

When run by not root, su's to root automatically.

This commit was sponsored by Brock Spratlen on Patreon.
---
 CHANGELOG                                   |  1 +
 Command/EnableTor.hs                        | 32 ++++++++++--
 Utility/Su.hs                               | 54 +++++++++++++++++++++
 doc/git-annex-enable-tor.mdwn               |  8 ++-
 doc/tips/peer_to_peer_network_with_tor.mdwn |  2 +-
 git-annex.cabal                             |  1 +
 6 files changed, 92 insertions(+), 6 deletions(-)
 create mode 100644 Utility/Su.hs

diff --git a/CHANGELOG b/CHANGELOG
index 65a3da82b3..220aeea41a 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -21,6 +21,7 @@ git-annex (6.20161211) UNRELEASED; urgency=medium
     present in the local repo even when it was not.
   * enable-tor: Put tor sockets in /var/lib/tor-annex/, rather
     than in /etc/tor/hidden_service/.
+  * enable-tor: No longer needs to be run as root.
   * Fix build with directory-1.3.
   * Debian: Suggest tor and magic-wormhole.
   * Debian: Build webapp on armel.
diff --git a/Command/EnableTor.hs b/Command/EnableTor.hs
index c6d477b4e7..91d5af701f 100644
--- a/Command/EnableTor.hs
+++ b/Command/EnableTor.hs
@@ -5,12 +5,20 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
+{-# LANGUAGE CPP #-}
+
 module Command.EnableTor where
 
 import Command
 import P2P.Address
 import Utility.Tor
 import Annex.UUID
+import Config.Files
+
+#ifndef mingw32_HOST_OS
+import Utility.Su
+import System.Posix.User
+#endif
 
 -- This runs as root, so avoid making any commits or initializing
 -- git-annex, or doing other things that create root-owned files.
@@ -23,9 +31,27 @@ seek :: CmdParams -> CommandSeek
 seek = withWords start
 
 start :: [String] -> CommandStart
-start ps = case readish =<< headMaybe ps of
-	Nothing -> giveup "Bad params"
-	Just userid -> do
+start os = do
+#ifndef mingw32_HOST_OS
+	curruserid <- liftIO getEffectiveUserID
+	if curruserid == 0
+		then case readish =<< headMaybe os of
+			Nothing -> giveup "Need user-id parameter."
+			Just userid -> go userid
+		else do
+			liftIO $ putStrLn "Need root access to enable tor..."
+			gitannex <- liftIO readProgramFile
+			let ps = [Param (cmdname cmd), Param (show curruserid)]
+			ifM (liftIO $ runAsRoot gitannex ps)
+				( stop
+				, giveup $ unwords $
+					[ "Failed to run as root:" , gitannex ] ++ toCommand ps
+				)
+#else
+	go 0
+#endif
+  where
+	go userid = do
 		uuid <- getUUID
 		when (uuid == NoUUID) $
 			giveup "This can only be run in a git-annex repository."
diff --git a/Utility/Su.hs b/Utility/Su.hs
new file mode 100644
index 0000000000..4244074d7a
--- /dev/null
+++ b/Utility/Su.hs
@@ -0,0 +1,54 @@
+{- su to root
+ -
+ - Copyright 2016 Joey Hess <id@joeyh.name>
+ -
+ - License: BSD-2-clause
+ -}
+
+module Utility.Su where
+
+import Common
+import Utility.Env
+import Utility.Path
+
+import System.Posix.Terminal
+
+-- Runs a command as root, fairly portably.
+--
+-- Does not use sudo commands if something else is available, because
+-- the user may not be in sudoers and we couldn't differentiate between
+-- that and the command failing. Although, some commands like gksu
+-- decide based on the system's configuration whether sudo should be used.
+runAsRoot :: String -> [CommandParam] -> IO Bool
+runAsRoot cmd ps = go =<< firstM (inPath . fst) =<< selectcmds
+  where
+	go Nothing = return False
+	go (Just (cmd', ps')) = boolSystem cmd' ps'
+
+	selectcmds = ifM (inx <||> (not <$> atconsole))
+		( return (graphicalcmds ++ consolecmds)
+		, return consolecmds
+		)
+	
+	inx = isJust <$> getEnv "DISPLAY"
+	atconsole = queryTerminal stdInput
+
+	-- These will only work when the user is logged into a desktop.
+	graphicalcmds =
+		[ ("gksu", [Param shellcmd])
+		, ("kdesu", [Param shellcmd])
+		-- Available in Debian's menu package; knows about lots of
+		-- ways to gain root.
+		, ("su-to-root", [Param "-X", Param "-c", Param shellcmd])
+		-- OSX native way to run a command as root, prompts in GUI
+		, ("osascript", [Param "-e", Param ("do shell script \"" ++ shellcmd ++ "\" with administrator privileges")])
+		]
+	
+	-- These will only work when run in a console.
+	consolecmds = 
+		[ ("su", [Param "-c", Param "--", Param cmd] ++ ps)
+		, ("sudo", [Param cmd] ++ ps)
+		, ("su-to-root", [Param "-c", Param shellcmd])
+		]
+	
+	shellcmd = unwords $ map shellEscape (cmd:toCommand ps)
diff --git a/doc/git-annex-enable-tor.mdwn b/doc/git-annex-enable-tor.mdwn
index 1c17380276..f06966400e 100644
--- a/doc/git-annex-enable-tor.mdwn
+++ b/doc/git-annex-enable-tor.mdwn
@@ -4,14 +4,18 @@ git-annex enable-tor - enable tor hidden service
 
 # SYNOPSIS
 
+git annex enable-tor
+
 sudo git annex enable-tor $(id -u)
 
 # DESCRIPTION
 
 This command enables a tor hidden service for git-annex.
 
-It has to be run by root, since it modifies `/etc/tor/torrc`.
-Pass it your user id number, as output by `id -u`
+It modifies `/etc/tor/torrc` to register the hidden service. If run as a
+normal user, it will try to use sudo/su/etc to get root access to modify
+that file. If you run it as root, pass it your non-root user id number,
+as output by `id -u`
 
 After this command is run, `git annex remotedaemon` can be run to serve the
 tor hidden service, and then `git-annex p2p --gen-address` can be run to
diff --git a/doc/tips/peer_to_peer_network_with_tor.mdwn b/doc/tips/peer_to_peer_network_with_tor.mdwn
index a57d8a5449..ce00b0424f 100644
--- a/doc/tips/peer_to_peer_network_with_tor.mdwn
+++ b/doc/tips/peer_to_peer_network_with_tor.mdwn
@@ -23,7 +23,7 @@ to accomplish this.
 
 In each git-annex repository, run these commands:
 
-	sudo git annex enable-tor $(id -u)
+	git annex enable-tor
 	git annex remotedaemon
 
 Now git-annex is running as a Tor hidden service, but 
diff --git a/git-annex.cabal b/git-annex.cabal
index 2f07c84377..81a6ac3ada 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1072,6 +1072,7 @@ Executable git-annex
     Utility.Shell
     Utility.SimpleProtocol
     Utility.SshConfig
+    Utility.Su
     Utility.SystemDirectory
     Utility.TList
     Utility.Tense

From 86401f84e23d438daefed9025fd00b1cdab70e86 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 20 Dec 2016 17:46:14 -0400
Subject: [PATCH 347/367] fail before suing when not in a git-annex repo

---
 Command/EnableTor.hs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/Command/EnableTor.hs b/Command/EnableTor.hs
index 91d5af701f..d12a6e446d 100644
--- a/Command/EnableTor.hs
+++ b/Command/EnableTor.hs
@@ -32,12 +32,15 @@ seek = withWords start
 
 start :: [String] -> CommandStart
 start os = do
+	uuid <- getUUID
+	when (uuid == NoUUID) $
+		giveup "This can only be run in a git-annex repository."
 #ifndef mingw32_HOST_OS
 	curruserid <- liftIO getEffectiveUserID
 	if curruserid == 0
 		then case readish =<< headMaybe os of
 			Nothing -> giveup "Need user-id parameter."
-			Just userid -> go userid
+			Just userid -> go uuid userid
 		else do
 			liftIO $ putStrLn "Need root access to enable tor..."
 			gitannex <- liftIO readProgramFile
@@ -48,13 +51,10 @@ start os = do
 					[ "Failed to run as root:" , gitannex ] ++ toCommand ps
 				)
 #else
-	go 0
+	go uuid 0
 #endif
   where
-	go userid = do
-		uuid <- getUUID
-		when (uuid == NoUUID) $
-			giveup "This can only be run in a git-annex repository."
+	go uuid userid = do
 		(onionaddr, onionport) <- liftIO $
 			addHiddenService "tor-annex" userid (fromUUID uuid)
 		storeP2PAddress $ TorAnnex onionaddr onionport

From f48b9775d8e4770e7428eeab9d4e381718778ad6 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Tue, 20 Dec 2016 17:46:30 -0400
Subject: [PATCH 348/367] cleanup

---
 Utility/Su.hs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Utility/Su.hs b/Utility/Su.hs
index 4244074d7a..001a09fbc6 100644
--- a/Utility/Su.hs
+++ b/Utility/Su.hs
@@ -9,7 +9,6 @@ module Utility.Su where
 
 import Common
 import Utility.Env
-import Utility.Path
 
 import System.Posix.Terminal
 

From a221b3fb1bad9d2ce8922d8ace9cb4c559c5b6fe Mon Sep 17 00:00:00 2001
From: "t.z.mates" <t.z.mates@web>
Date: Tue, 20 Dec 2016 23:08:44 +0000
Subject: [PATCH 349/367] Added a comment

---
 ...ment_4_f9d6dffb2617715c58216f54016de3a4._comment | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_4_f9d6dffb2617715c58216f54016de3a4._comment

diff --git a/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_4_f9d6dffb2617715c58216f54016de3a4._comment b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_4_f9d6dffb2617715c58216f54016de3a4._comment
new file mode 100644
index 0000000000..af0b2030b9
--- /dev/null
+++ b/doc/bugs/add_fails_with_v6_repo_when_four_levels_deep/comment_4_f9d6dffb2617715c58216f54016de3a4._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="t.z.mates"
+ avatar="http://cdn.libravatar.org/avatar/90f15fad216078fd08d62cc676487925"
+ subject="comment 4"
+ date="2016-12-20T23:08:44Z"
+ content="""
+Hmm, I don't think an old version of git is the cause. I'm currently running the most recent build of git (2.11.0), but have used a number of versions over the past year.
+
+I'm not sure if this is relevant, but this other bug reports similar behavior: [sync --content, fatal is outside repository errors](https://git-annex.branchable.com/forum/sync_--content__44___fatal_is_outside_repository_errors/). Specifically, it notes that there is an odd use of relative paths:
+> The relative path ../Users is curious
+
+My error also appends an extra period. In particular, the path should be \"./1/2/3/4/foo\" but prints \"../1/2/3/4/foo\".
+"""]]

From babfe5acbb9632f83317235c60cb9bbd895937cf Mon Sep 17 00:00:00 2001
From: "tom@6463e5ce4c0e478cb7583fb37288d6954e17bfc6" <tom@web>
Date: Wed, 21 Dec 2016 04:18:13 +0000
Subject: [PATCH 350/367]

---
 ...ares_treated_as_a_crippled_filesystem.mdwn | 46 +++++++++++++++++++
 1 file changed, 46 insertions(+)
 create mode 100644 doc/bugs/unRAID_shares_treated_as_a_crippled_filesystem.mdwn

diff --git a/doc/bugs/unRAID_shares_treated_as_a_crippled_filesystem.mdwn b/doc/bugs/unRAID_shares_treated_as_a_crippled_filesystem.mdwn
new file mode 100644
index 0000000000..dd491e5fda
--- /dev/null
+++ b/doc/bugs/unRAID_shares_treated_as_a_crippled_filesystem.mdwn
@@ -0,0 +1,46 @@
+### Please describe the problem.
+
+Running `git annex init` on an [unRAID server](https://lime-technology.com/what-is-unraid/) results in an annex created with `crippledfilesystem = true` and `direct = true`. I understand from reading [this](https://git-annex.branchable.com/design/assistant/blog/day_188__crippled_filesystem_support/) that it occurs when `git annex init` performs a probe to determine if all of the following are supported:
+
+1. symlinks
+2. hard links
+3. unix permissions
+
+Although unRAID disks are formatted with xfs, and therefore support all three of the above, I'm assuming that unRAID's method of combining multiple disks into one "share" is the cause of the problem (hardlinks still work on a single disk, but not on shares that span multiple disks). Symlinks and unix permissions work normally in the unRAID-created shares.
+
+Is there any way to allow the use of 'indirect' mode with multi-disk shares? As I mentioned, symlinks and unix permissions work normally--it's only the hardlinks that won't work across the multi-disk shares.
+
+I can create a 'normal' annex as long as I `cd` to a single disk drive first--what would happen if the annex was later moved onto a multi-disk share? Would it still work? Would it fail gracefully? Would it cause data loss?
+
+### What steps will reproduce the problem?
+
+    cd /mnt/user/NameOfShare
+    git init
+    git annex init
+
+The following will result in the creation of a 'normal' indirect share:
+
+    cd /mnt/disk1
+    git init
+    git annex init
+
+### What version of git-annex are you using? On what operating system?
+
+    git-annex version: 6.20161211-gc3ab3c668
+    build flags: Assistant Webapp Pairing Testsuite S3(multipartupload)(storageclasses) WebDAV Inotify DBus DesktopNotify XMPP ConcurrentOutput TorrentParser MagicMime Feeds Quvi
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 SHA1E SHA1 MD5E MD5 WORM URL
+    remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav tahoe glacier ddar hook external
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Has been working great, so far, except for the above.

From 28307391339ad0f83bf09ac7d749440d4a664160 Mon Sep 17 00:00:00 2001
From: marekj <marekj@web>
Date: Wed, 21 Dec 2016 12:12:42 +0000
Subject: [PATCH 351/367]

---
 doc/forum/What_is_the_assistant_up_to__63__.mdwn | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 doc/forum/What_is_the_assistant_up_to__63__.mdwn

diff --git a/doc/forum/What_is_the_assistant_up_to__63__.mdwn b/doc/forum/What_is_the_assistant_up_to__63__.mdwn
new file mode 100644
index 0000000000..623268f7e4
--- /dev/null
+++ b/doc/forum/What_is_the_assistant_up_to__63__.mdwn
@@ -0,0 +1,5 @@
+Is there a way to see what the assistant is doing right now, what failed, etc. ?
+I am running the assistant on a remote server so the Webapp's interface is not easily available.
+
+I was hoping to have something that is easier to read than the daemon.log
+

From 6eeb07a9c14b46df980ffcbd24bba9483f1d28e5 Mon Sep 17 00:00:00 2001
From: marekj <marekj@web>
Date: Wed, 21 Dec 2016 12:58:37 +0000
Subject: [PATCH 352/367] Added a comment: git annex info

---
 .../comment_1_9baa0e54c19105c7cce946c19c587866._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/What_is_the_assistant_up_to__63__/comment_1_9baa0e54c19105c7cce946c19c587866._comment

diff --git a/doc/forum/What_is_the_assistant_up_to__63__/comment_1_9baa0e54c19105c7cce946c19c587866._comment b/doc/forum/What_is_the_assistant_up_to__63__/comment_1_9baa0e54c19105c7cce946c19c587866._comment
new file mode 100644
index 0000000000..f007f5d301
--- /dev/null
+++ b/doc/forum/What_is_the_assistant_up_to__63__/comment_1_9baa0e54c19105c7cce946c19c587866._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="marekj"
+ avatar="http://cdn.libravatar.org/avatar/65a60e8f5183feeeef8cef815bf73e61"
+ subject="git annex info"
+ date="2016-12-21T12:58:37Z"
+ content="""
+I found that git annex info provides information on current transfers. Using -F it provides what I want.
+"""]]

From 872064e4a6eda48f884aad49eb9f4628d437e1be Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 21 Dec 2016 14:21:05 -0400
Subject: [PATCH 353/367] comment

---
 ...comment_6_640e5c6cdea8a6fae63c3fab6970f1f2._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/todo/Workflow_guide/comment_6_640e5c6cdea8a6fae63c3fab6970f1f2._comment

diff --git a/doc/todo/Workflow_guide/comment_6_640e5c6cdea8a6fae63c3fab6970f1f2._comment b/doc/todo/Workflow_guide/comment_6_640e5c6cdea8a6fae63c3fab6970f1f2._comment
new file mode 100644
index 0000000000..9eae8e9119
--- /dev/null
+++ b/doc/todo/Workflow_guide/comment_6_640e5c6cdea8a6fae63c3fab6970f1f2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2016-12-21T18:19:07Z"
+ content="""
+In a way the use cases on the front page of the website are trying to
+accomplish the same thing requested here. I think that section could be
+moved more in the direction of listing some ways to use git-annex and
+linking to walkthroughs for the different use cases.
+"""]]

From 405fbd25e19bcfa754aa081fc7b52e7e6dd37f9b Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 21 Dec 2016 14:31:27 -0400
Subject: [PATCH 354/367] include tor-annex in hidden service directory names

To make it easier to manage/delete them etc.

Backwards compatablity is preserved for existing tor configs.
---
 Command/EnableTor.hs          |  2 +-
 P2P/Address.hs                |  3 +++
 RemoteDaemon/Transport/Tor.hs |  2 +-
 Utility/Tor.hs                | 27 +++++++++++++++------------
 4 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/Command/EnableTor.hs b/Command/EnableTor.hs
index d12a6e446d..c81968a559 100644
--- a/Command/EnableTor.hs
+++ b/Command/EnableTor.hs
@@ -56,6 +56,6 @@ start os = do
   where
 	go uuid userid = do
 		(onionaddr, onionport) <- liftIO $
-			addHiddenService "tor-annex" userid (fromUUID uuid)
+			addHiddenService torAppName userid (fromUUID uuid)
 		storeP2PAddress $ TorAnnex onionaddr onionport
 		stop
diff --git a/P2P/Address.hs b/P2P/Address.hs
index 1b1f66059e..d911f7b4b2 100644
--- a/P2P/Address.hs
+++ b/P2P/Address.hs
@@ -90,3 +90,6 @@ storeP2PAddress addr = do
 
 p2pAddressCredsFile :: FilePath
 p2pAddressCredsFile = "p2paddrs"
+
+torAppName :: AppName
+torAppName = "tor-annex"
diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs
index 3f70fb1fbb..0fbe9a7200 100644
--- a/RemoteDaemon/Transport/Tor.hs
+++ b/RemoteDaemon/Transport/Tor.hs
@@ -41,7 +41,7 @@ server th@(TransportHandle (LocalRepo r) _) = do
 	u <- liftAnnex th getUUID
 	uid <- getRealUserID
 	let ident = fromUUID u
-	go u =<< getHiddenServiceSocketFile uid ident
+	go u =<< getHiddenServiceSocketFile torAppName uid ident
   where
 	go u (Just sock) = do
 		q <- newTBMQueueIO maxConnections
diff --git a/Utility/Tor.hs b/Utility/Tor.hs
index 64a6ae11da..4e7c0ef43f 100644
--- a/Utility/Tor.hs
+++ b/Utility/Tor.hs
@@ -66,7 +66,7 @@ addHiddenService appname uid ident = do
 			writeFile torrc $ unlines $
 				ls ++
 				[ ""
-				, "HiddenServiceDir " ++ hiddenServiceDir uid ident
+				, "HiddenServiceDir " ++ hiddenServiceDir appname uid ident
 				, "HiddenServicePort " ++ show newport ++ 
 					" unix:" ++ sockfile
 				]
@@ -95,7 +95,7 @@ addHiddenService appname uid ident = do
 	waithiddenservice :: Int -> OnionPort -> IO (OnionAddress, OnionPort)
 	waithiddenservice 0 _ = giveup "tor failed to create hidden service, perhaps the tor service is not running"
 	waithiddenservice n p = do
-		v <- tryIO $ readFile $ hiddenServiceHostnameFile uid ident
+		v <- tryIO $ readFile $ hiddenServiceHostnameFile appname uid ident
 		case v of
 			Right s | ".onion\n" `isSuffixOf` s ->
 				return (OnionAddress (takeWhile (/= '\n') s), p)
@@ -105,13 +105,14 @@ addHiddenService appname uid ident = do
 
 -- | A hidden service directory to use.
 --
--- The "hs" is used in the name to prevent too long a path name,
--- which could present problems for socketFile.
-hiddenServiceDir :: UserID -> UniqueIdent -> FilePath
-hiddenServiceDir uid ident = torLibDir </> "hs_" ++ show uid ++ "_" ++ ident
+-- Has to be inside the torLibDir so tor can create it.
+--
+-- Has to end with "uid_ident" so getHiddenServiceSocketFile can find it.
+hiddenServiceDir :: AppName -> UserID -> UniqueIdent -> FilePath
+hiddenServiceDir appname uid ident = torLibDir </> appname ++ "_" ++ show uid ++ "_" ++ ident
 
-hiddenServiceHostnameFile :: UserID -> UniqueIdent -> FilePath
-hiddenServiceHostnameFile uid ident = hiddenServiceDir uid ident </> "hostname"
+hiddenServiceHostnameFile :: AppName -> UserID -> UniqueIdent -> FilePath
+hiddenServiceHostnameFile appname uid ident = hiddenServiceDir appname uid ident </> "hostname"
 
 -- | Location of the socket for a hidden service.
 --
@@ -126,18 +127,20 @@ hiddenServiceSocketFile appname uid ident = varLibDir </> appname </> show uid +
 
 -- | Parse torrc, to get the socket file used for a hidden service with
 -- the specified UniqueIdent.
-getHiddenServiceSocketFile :: UserID -> UniqueIdent -> IO (Maybe FilePath)
-getHiddenServiceSocketFile uid ident = 
+getHiddenServiceSocketFile :: AppName -> UserID -> UniqueIdent -> IO (Maybe FilePath)
+getHiddenServiceSocketFile _appname uid ident = 
 	parse . map words . lines <$> catchDefaultIO "" (readFile torrc)
   where
 	parse [] = Nothing
 	parse (("HiddenServiceDir":hsdir:[]):("HiddenServicePort":_hsport:hsaddr:[]):rest)
-		| "unix:" `isPrefixOf` hsaddr && hsdir == hsdir_want =
+		| "unix:" `isPrefixOf` hsaddr && hasident hsdir =
 			Just (drop (length "unix:") hsaddr)
 		| otherwise = parse rest
 	parse (_:rest) = parse rest
 
-	hsdir_want = hiddenServiceDir uid ident
+	-- Don't look for AppName in the hsdir, because it didn't used to
+	-- be included.
+	hasident hsdir = (show uid ++ "_" ++ ident) `isSuffixOf` takeFileName hsdir
 
 -- | Sets up the directory for the socketFile, with appropriate
 -- permissions. Must run as root.

From 942ea305a23724b0345b645f4f7d8ae77069623c Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Wed, 21 Dec 2016 15:11:10 -0400
Subject: [PATCH 355/367] todo

---
 doc/todo/tor.mdwn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index cb0bc4d410..f0c1936776 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -4,6 +4,7 @@ Mostly working!
 
 Current todo list:
 
+* Make enable-tor check connection back to itself to verify tor is working.
 * When a transfer can't be done because another transfer of the same 
   object is already in progress, the message about this is output by the
   remotedaemon --debug, but not forwarded to the peer, which shows

From 458a786c4d4882a2aad3c0ecf7298f337c9436a3 Mon Sep 17 00:00:00 2001
From: "http://svario.it/gioele" <Gioele@web>
Date: Wed, 21 Dec 2016 21:51:09 +0000
Subject: [PATCH 356/367] Added a comment: compiled with GHC 8, but LOCPATH is
 still set

---
 ...ment_5_eca31aeb974571c9cca7a399e00984a5._comment | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/bugs/cannot_change_locale___40__en__95__US.UTF-8__41__/comment_5_eca31aeb974571c9cca7a399e00984a5._comment

diff --git a/doc/bugs/cannot_change_locale___40__en__95__US.UTF-8__41__/comment_5_eca31aeb974571c9cca7a399e00984a5._comment b/doc/bugs/cannot_change_locale___40__en__95__US.UTF-8__41__/comment_5_eca31aeb974571c9cca7a399e00984a5._comment
new file mode 100644
index 0000000000..9cf3695f58
--- /dev/null
+++ b/doc/bugs/cannot_change_locale___40__en__95__US.UTF-8__41__/comment_5_eca31aeb974571c9cca7a399e00984a5._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://svario.it/gioele"
+ nickname="Gioele"
+ avatar="http://cdn.libravatar.org/avatar/af2f2ba0dafe4650011d20f2168d43cff773aba97f55ae5b252bb873f391c1e2"
+ subject="compiled with GHC 8, but LOCPATH is still set"
+ date="2016-12-21T21:51:09Z"
+ content="""
+This bug does not want to die.
+
+The current standalone build (`6.20161211-gc3ab3c668`) has been compiled with GHC 8 but when I launch `runshell`, I still see that `LOCPATH` is set and the character encoding is messed up.
+
+I deduced the version of GHC used to compile git-annex with `strings ./shimmed/git-annex/git-annex | grep 'GHC [0-9]'`.
+"""]]

From 3aaabc906b776075e190739b42157959b4e09f31 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Thu, 22 Dec 2016 13:59:21 -0400
Subject: [PATCH 357/367] close

---
 CHANGELOG                                   |  2 ++
 Command/EnableTor.hs                        | 36 ++++++++++++++++++---
 P2P/IO.hs                                   | 29 +++++++++++++++++
 RemoteDaemon/Transport/Tor.hs               | 16 +--------
 doc/bugs/YouTube_-_error_in_importfeed.mdwn |  3 ++
 doc/todo/tor.mdwn                           |  1 -
 6 files changed, 67 insertions(+), 20 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index 220aeea41a..2d7ea22a70 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -22,6 +22,8 @@ git-annex (6.20161211) UNRELEASED; urgency=medium
   * enable-tor: Put tor sockets in /var/lib/tor-annex/, rather
     than in /etc/tor/hidden_service/.
   * enable-tor: No longer needs to be run as root.
+  * enable-tor: When run as a regular user, test a connection back to 
+    the hidden service over tor.
   * Fix build with directory-1.3.
   * Debian: Suggest tor and magic-wormhole.
   * Debian: Build webapp on armel.
diff --git a/Command/EnableTor.hs b/Command/EnableTor.hs
index c81968a559..2b7d626351 100644
--- a/Command/EnableTor.hs
+++ b/Command/EnableTor.hs
@@ -10,18 +10,19 @@
 module Command.EnableTor where
 
 import Command
+import qualified Annex
 import P2P.Address
 import Utility.Tor
 import Annex.UUID
 import Config.Files
+import P2P.IO
+import Utility.ThreadScheduler
 
 #ifndef mingw32_HOST_OS
 import Utility.Su
 import System.Posix.User
 #endif
 
--- This runs as root, so avoid making any commits or initializing
--- git-annex, or doing other things that create root-owned files.
 cmd :: Command
 cmd = noCommit $ dontCheck repoExists $
 	command "enable-tor" SectionSetup "enable tor hidden service"
@@ -30,6 +31,8 @@ cmd = noCommit $ dontCheck repoExists $
 seek :: CmdParams -> CommandSeek
 seek = withWords start
 
+-- This runs as root, so avoid making any commits or initializing
+-- git-annex, or doing other things that create root-owned files.
 start :: [String] -> CommandStart
 start os = do
 	uuid <- getUUID
@@ -42,11 +45,12 @@ start os = do
 			Nothing -> giveup "Need user-id parameter."
 			Just userid -> go uuid userid
 		else do
-			liftIO $ putStrLn "Need root access to enable tor..."
+			showStart "enable-tor" ""
+			showLongNote "Need root access to enable tor..."
 			gitannex <- liftIO readProgramFile
 			let ps = [Param (cmdname cmd), Param (show curruserid)]
 			ifM (liftIO $ runAsRoot gitannex ps)
-				( stop
+				( next $ next checkHiddenService
 				, giveup $ unwords $
 					[ "Failed to run as root:" , gitannex ] ++ toCommand ps
 				)
@@ -59,3 +63,27 @@ start os = do
 			addHiddenService torAppName userid (fromUUID uuid)
 		storeP2PAddress $ TorAnnex onionaddr onionport
 		stop
+
+checkHiddenService :: CommandCleanup
+checkHiddenService = do
+	showLongNote "Tor hidden service is configured. Checking connection to it. This may take a few minutes."
+	go (150 :: Int) =<< filter istoraddr <$> loadP2PAddresses
+  where
+	istoraddr (TorAnnex _ _) = True
+
+	go 0 _ = giveup "Still unable to connect to hidden service. It might not yet be usable by others. Please check Tor's logs for details."
+	go _ [] = giveup "Somehow didn't get an onion address."
+	go n addrs@(addr:_) = do
+		g <- Annex.gitRepo
+		-- Connect to ourselves; don't bother trying to auth,
+		-- we just want to know if the circuit works.
+		cv <- liftIO $ tryNonAsync $ connectPeer g addr
+		case cv of
+			Left e -> do
+				warning $ "Unable to connect to hidden service. It may not yet have propigated to the Tor network. (" ++ show e ++ ") Will retry.."
+				liftIO $ threadDelaySeconds (Seconds 2)
+				go (n-1) addrs
+			Right conn -> do
+				liftIO $ closeConnection conn
+				showLongNote "Tor hidden service is working."
+				return True
diff --git a/P2P/IO.hs b/P2P/IO.hs
index 3e0999775f..49f28bff1d 100644
--- a/P2P/IO.hs
+++ b/P2P/IO.hs
@@ -68,6 +68,35 @@ closeConnection conn = do
 	hClose (connIhdl conn)
 	hClose (connOhdl conn)
 
+-- Serves the protocol on a unix socket.
+--
+-- The callback is run to serve a connection, and is responsible for
+-- closing the Handle when done.
+--
+-- Note that while the callback is running, other connections won't be
+-- processes, so longterm work should be run in a separate thread by
+-- the callback.
+serveUnixSocket :: FilePath -> (Handle -> IO ()) -> IO ()
+serveUnixSocket unixsocket serveconn = do
+	nukeFile unixsocket
+	soc <- S.socket S.AF_UNIX S.Stream S.defaultProtocol
+	S.bind soc (S.SockAddrUnix unixsocket)
+	-- Allow everyone to read and write to the socket,
+	-- so a daemon like tor, that is probably running as a different
+	-- de sock $ addModes
+	-- user, can access it.
+	-- 
+        -- Connections have to authenticate to do anything,
+        -- so it's fine that other local users can connect to the
+        -- socket.
+	modifyFileMode unixsocket $ addModes
+		[groupReadMode, groupWriteMode, otherReadMode, otherWriteMode]
+	S.listen soc 2
+	forever $ do
+		(conn, _) <- S.accept soc
+		h <- setupHandle conn
+		serveconn conn
+
 setupHandle :: Socket -> IO Handle
 setupHandle s = do
 	h <- socketToHandle s ReadWriteMode
diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs
index 0fbe9a7200..43ff3a2c1b 100644
--- a/RemoteDaemon/Transport/Tor.hs
+++ b/RemoteDaemon/Transport/Tor.hs
@@ -48,22 +48,8 @@ server th@(TransportHandle (LocalRepo r) _) = do
 		replicateM_ maxConnections $
 			forkIO $ forever $ serveClient th u r q
 
-		nukeFile sock
-		soc <- S.socket S.AF_UNIX S.Stream S.defaultProtocol
-		S.bind soc (S.SockAddrUnix sock)
-		-- Allow everyone to read and write to the socket; tor
-		-- is probably running as a different user.
-		-- Connections have to authenticate to do anything,
-		-- so it's fine that other local users can connect to the
-		-- socket.
-		modifyFileMode sock $ addModes
-			[groupReadMode, groupWriteMode, otherReadMode, otherWriteMode]
-
-		S.listen soc 2
 		debugM "remotedaemon" "Tor hidden service running"
-		forever $ do
-			(conn, _) <- S.accept soc
-			h <- setupHandle conn
+		serveUnixSocket sock $ \conn -> do
 			ok <- atomically $ ifM (isFullTBMQueue q)
 				( return False
 				, do
diff --git a/doc/bugs/YouTube_-_error_in_importfeed.mdwn b/doc/bugs/YouTube_-_error_in_importfeed.mdwn
index b02348f656..d300c621f4 100644
--- a/doc/bugs/YouTube_-_error_in_importfeed.mdwn
+++ b/doc/bugs/YouTube_-_error_in_importfeed.mdwn
@@ -69,3 +69,6 @@ ok
 ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
 
 Yes, for years.  I donated to fund the dev and proudly display my git-annex stickers!
+
+> This is now fixed in feed's git repository, and will be in the next
+> release of feed after the current 0.3.11.1 release. [[done]] --[[Joey]]
diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index f0c1936776..cb0bc4d410 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -4,7 +4,6 @@ Mostly working!
 
 Current todo list:
 
-* Make enable-tor check connection back to itself to verify tor is working.
 * When a transfer can't be done because another transfer of the same 
   object is already in progress, the message about this is output by the
   remotedaemon --debug, but not forwarded to the peer, which shows

From 22252e8e4c1bc17b9ce63f3e69c4ac73f1412969 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 24 Dec 2016 12:07:15 -0400
Subject: [PATCH 358/367] Revert "close"

This reverts commit 3aaabc906b776075e190739b42157959b4e09f31.

Commit contained incomplete work.
---
 CHANGELOG                                   |  2 --
 Command/EnableTor.hs                        | 36 +++------------------
 P2P/IO.hs                                   | 29 -----------------
 RemoteDaemon/Transport/Tor.hs               | 16 ++++++++-
 doc/bugs/YouTube_-_error_in_importfeed.mdwn |  3 --
 doc/todo/tor.mdwn                           |  1 +
 6 files changed, 20 insertions(+), 67 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index 2d7ea22a70..220aeea41a 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -22,8 +22,6 @@ git-annex (6.20161211) UNRELEASED; urgency=medium
   * enable-tor: Put tor sockets in /var/lib/tor-annex/, rather
     than in /etc/tor/hidden_service/.
   * enable-tor: No longer needs to be run as root.
-  * enable-tor: When run as a regular user, test a connection back to 
-    the hidden service over tor.
   * Fix build with directory-1.3.
   * Debian: Suggest tor and magic-wormhole.
   * Debian: Build webapp on armel.
diff --git a/Command/EnableTor.hs b/Command/EnableTor.hs
index 2b7d626351..c81968a559 100644
--- a/Command/EnableTor.hs
+++ b/Command/EnableTor.hs
@@ -10,19 +10,18 @@
 module Command.EnableTor where
 
 import Command
-import qualified Annex
 import P2P.Address
 import Utility.Tor
 import Annex.UUID
 import Config.Files
-import P2P.IO
-import Utility.ThreadScheduler
 
 #ifndef mingw32_HOST_OS
 import Utility.Su
 import System.Posix.User
 #endif
 
+-- This runs as root, so avoid making any commits or initializing
+-- git-annex, or doing other things that create root-owned files.
 cmd :: Command
 cmd = noCommit $ dontCheck repoExists $
 	command "enable-tor" SectionSetup "enable tor hidden service"
@@ -31,8 +30,6 @@ cmd = noCommit $ dontCheck repoExists $
 seek :: CmdParams -> CommandSeek
 seek = withWords start
 
--- This runs as root, so avoid making any commits or initializing
--- git-annex, or doing other things that create root-owned files.
 start :: [String] -> CommandStart
 start os = do
 	uuid <- getUUID
@@ -45,12 +42,11 @@ start os = do
 			Nothing -> giveup "Need user-id parameter."
 			Just userid -> go uuid userid
 		else do
-			showStart "enable-tor" ""
-			showLongNote "Need root access to enable tor..."
+			liftIO $ putStrLn "Need root access to enable tor..."
 			gitannex <- liftIO readProgramFile
 			let ps = [Param (cmdname cmd), Param (show curruserid)]
 			ifM (liftIO $ runAsRoot gitannex ps)
-				( next $ next checkHiddenService
+				( stop
 				, giveup $ unwords $
 					[ "Failed to run as root:" , gitannex ] ++ toCommand ps
 				)
@@ -63,27 +59,3 @@ start os = do
 			addHiddenService torAppName userid (fromUUID uuid)
 		storeP2PAddress $ TorAnnex onionaddr onionport
 		stop
-
-checkHiddenService :: CommandCleanup
-checkHiddenService = do
-	showLongNote "Tor hidden service is configured. Checking connection to it. This may take a few minutes."
-	go (150 :: Int) =<< filter istoraddr <$> loadP2PAddresses
-  where
-	istoraddr (TorAnnex _ _) = True
-
-	go 0 _ = giveup "Still unable to connect to hidden service. It might not yet be usable by others. Please check Tor's logs for details."
-	go _ [] = giveup "Somehow didn't get an onion address."
-	go n addrs@(addr:_) = do
-		g <- Annex.gitRepo
-		-- Connect to ourselves; don't bother trying to auth,
-		-- we just want to know if the circuit works.
-		cv <- liftIO $ tryNonAsync $ connectPeer g addr
-		case cv of
-			Left e -> do
-				warning $ "Unable to connect to hidden service. It may not yet have propigated to the Tor network. (" ++ show e ++ ") Will retry.."
-				liftIO $ threadDelaySeconds (Seconds 2)
-				go (n-1) addrs
-			Right conn -> do
-				liftIO $ closeConnection conn
-				showLongNote "Tor hidden service is working."
-				return True
diff --git a/P2P/IO.hs b/P2P/IO.hs
index 49f28bff1d..3e0999775f 100644
--- a/P2P/IO.hs
+++ b/P2P/IO.hs
@@ -68,35 +68,6 @@ closeConnection conn = do
 	hClose (connIhdl conn)
 	hClose (connOhdl conn)
 
--- Serves the protocol on a unix socket.
---
--- The callback is run to serve a connection, and is responsible for
--- closing the Handle when done.
---
--- Note that while the callback is running, other connections won't be
--- processes, so longterm work should be run in a separate thread by
--- the callback.
-serveUnixSocket :: FilePath -> (Handle -> IO ()) -> IO ()
-serveUnixSocket unixsocket serveconn = do
-	nukeFile unixsocket
-	soc <- S.socket S.AF_UNIX S.Stream S.defaultProtocol
-	S.bind soc (S.SockAddrUnix unixsocket)
-	-- Allow everyone to read and write to the socket,
-	-- so a daemon like tor, that is probably running as a different
-	-- de sock $ addModes
-	-- user, can access it.
-	-- 
-        -- Connections have to authenticate to do anything,
-        -- so it's fine that other local users can connect to the
-        -- socket.
-	modifyFileMode unixsocket $ addModes
-		[groupReadMode, groupWriteMode, otherReadMode, otherWriteMode]
-	S.listen soc 2
-	forever $ do
-		(conn, _) <- S.accept soc
-		h <- setupHandle conn
-		serveconn conn
-
 setupHandle :: Socket -> IO Handle
 setupHandle s = do
 	h <- socketToHandle s ReadWriteMode
diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs
index 43ff3a2c1b..0fbe9a7200 100644
--- a/RemoteDaemon/Transport/Tor.hs
+++ b/RemoteDaemon/Transport/Tor.hs
@@ -48,8 +48,22 @@ server th@(TransportHandle (LocalRepo r) _) = do
 		replicateM_ maxConnections $
 			forkIO $ forever $ serveClient th u r q
 
+		nukeFile sock
+		soc <- S.socket S.AF_UNIX S.Stream S.defaultProtocol
+		S.bind soc (S.SockAddrUnix sock)
+		-- Allow everyone to read and write to the socket; tor
+		-- is probably running as a different user.
+		-- Connections have to authenticate to do anything,
+		-- so it's fine that other local users can connect to the
+		-- socket.
+		modifyFileMode sock $ addModes
+			[groupReadMode, groupWriteMode, otherReadMode, otherWriteMode]
+
+		S.listen soc 2
 		debugM "remotedaemon" "Tor hidden service running"
-		serveUnixSocket sock $ \conn -> do
+		forever $ do
+			(conn, _) <- S.accept soc
+			h <- setupHandle conn
 			ok <- atomically $ ifM (isFullTBMQueue q)
 				( return False
 				, do
diff --git a/doc/bugs/YouTube_-_error_in_importfeed.mdwn b/doc/bugs/YouTube_-_error_in_importfeed.mdwn
index d300c621f4..b02348f656 100644
--- a/doc/bugs/YouTube_-_error_in_importfeed.mdwn
+++ b/doc/bugs/YouTube_-_error_in_importfeed.mdwn
@@ -69,6 +69,3 @@ ok
 ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
 
 Yes, for years.  I donated to fund the dev and proudly display my git-annex stickers!
-
-> This is now fixed in feed's git repository, and will be in the next
-> release of feed after the current 0.3.11.1 release. [[done]] --[[Joey]]
diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index cb0bc4d410..f0c1936776 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -4,6 +4,7 @@ Mostly working!
 
 Current todo list:
 
+* Make enable-tor check connection back to itself to verify tor is working.
 * When a transfer can't be done because another transfer of the same 
   object is already in progress, the message about this is output by the
   remotedaemon --debug, but not forwarded to the peer, which shows

From fd0b9aaa57644def3552419e7aefb00519672ab6 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 24 Dec 2016 12:07:50 -0400
Subject: [PATCH 359/367] close

---
 doc/bugs/YouTube_-_error_in_importfeed.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/bugs/YouTube_-_error_in_importfeed.mdwn b/doc/bugs/YouTube_-_error_in_importfeed.mdwn
index b02348f656..d300c621f4 100644
--- a/doc/bugs/YouTube_-_error_in_importfeed.mdwn
+++ b/doc/bugs/YouTube_-_error_in_importfeed.mdwn
@@ -69,3 +69,6 @@ ok
 ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
 
 Yes, for years.  I donated to fund the dev and proudly display my git-annex stickers!
+
+> This is now fixed in feed's git repository, and will be in the next
+> release of feed after the current 0.3.11.1 release. [[done]] --[[Joey]]

From f3a4b9191c65a1dcab78fcce351ff023523a7048 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 24 Dec 2016 12:12:58 -0400
Subject: [PATCH 360/367] refactor

---
 P2P/IO.hs                     | 38 +++++++++++++++++++++++++++++------
 RemoteDaemon/Transport/Tor.hs | 22 +++-----------------
 2 files changed, 35 insertions(+), 25 deletions(-)

diff --git a/P2P/IO.hs b/P2P/IO.hs
index 3e0999775f..89f712fcae 100644
--- a/P2P/IO.hs
+++ b/P2P/IO.hs
@@ -12,35 +12,33 @@ module P2P.IO
 	, P2PConnection(..)
 	, connectPeer
 	, closeConnection
+	, serveUnixSocket
 	, setupHandle
 	, runNetProto
 	, runNet
 	) where
 
+import Common
 import P2P.Protocol
 import P2P.Address
-import Utility.Process
 import Git
 import Git.Command
 import Utility.AuthToken
-import Utility.SafeCommand
 import Utility.SimpleProtocol
-import Utility.Exception
 import Utility.Metered
 import Utility.Tor
-import Utility.FileSystemEncoding
+import Utility.FileMode
 
-import Control.Monad
 import Control.Monad.Free
 import Control.Monad.IO.Class
 import System.Exit (ExitCode(..))
 import Network.Socket
-import System.IO
 import Control.Concurrent
 import Control.Concurrent.Async
 import qualified Data.ByteString as B
 import qualified Data.ByteString.Lazy as L
 import System.Log.Logger (debugM)
+import qualified Network.Socket as S
 
 -- Type of interpreters of the Proto free monad.
 type RunProto m = forall a. (MonadIO m, MonadMask m) => Proto a -> m (Either String a)
@@ -68,6 +66,34 @@ closeConnection conn = do
 	hClose (connIhdl conn)
 	hClose (connOhdl conn)
 
+-- Serves the protocol on a unix socket.
+--
+-- The callback is run to serve a connection, and is responsible for
+-- closing the Handle when done.
+--
+-- Note that while the callback is running, other connections won't be
+-- processes, so longterm work should be run in a separate thread by
+-- the callback.
+serveUnixSocket :: FilePath -> (Handle -> IO ()) -> IO ()
+serveUnixSocket unixsocket serveconn = do
+	nukeFile unixsocket
+	soc <- S.socket S.AF_UNIX S.Stream S.defaultProtocol
+	S.bind soc (S.SockAddrUnix unixsocket)
+	-- Allow everyone to read and write to the socket,
+	-- so a daemon like tor, that is probably running as a different
+	-- de sock $ addModes
+	-- user, can access it.
+	-- 
+        -- Connections have to authenticate to do anything,
+        -- so it's fine that other local users can connect to the
+        -- socket.
+	modifyFileMode unixsocket $ addModes
+		[groupReadMode, groupWriteMode, otherReadMode, otherWriteMode]
+	S.listen soc 2
+	forever $ do
+		(conn, _) <- S.accept soc
+		setupHandle conn >>= serveconn
+
 setupHandle :: Socket -> IO Handle
 setupHandle s = do
 	h <- socketToHandle s ReadWriteMode
diff --git a/RemoteDaemon/Transport/Tor.hs b/RemoteDaemon/Transport/Tor.hs
index 0fbe9a7200..e7d3794d66 100644
--- a/RemoteDaemon/Transport/Tor.hs
+++ b/RemoteDaemon/Transport/Tor.hs
@@ -14,7 +14,6 @@ import Annex.ChangedRefs
 import RemoteDaemon.Types
 import RemoteDaemon.Common
 import Utility.Tor
-import Utility.FileMode
 import Utility.AuthToken
 import P2P.Protocol as P2P
 import P2P.IO
@@ -33,7 +32,6 @@ import System.Log.Logger (debugM)
 import Control.Concurrent.STM
 import Control.Concurrent.STM.TBMQueue
 import Control.Concurrent.Async
-import qualified Network.Socket as S
 
 -- Run tor hidden service.
 server :: TransportHandle -> IO ()
@@ -48,30 +46,16 @@ server th@(TransportHandle (LocalRepo r) _) = do
 		replicateM_ maxConnections $
 			forkIO $ forever $ serveClient th u r q
 
-		nukeFile sock
-		soc <- S.socket S.AF_UNIX S.Stream S.defaultProtocol
-		S.bind soc (S.SockAddrUnix sock)
-		-- Allow everyone to read and write to the socket; tor
-		-- is probably running as a different user.
-		-- Connections have to authenticate to do anything,
-		-- so it's fine that other local users can connect to the
-		-- socket.
-		modifyFileMode sock $ addModes
-			[groupReadMode, groupWriteMode, otherReadMode, otherWriteMode]
-
-		S.listen soc 2
 		debugM "remotedaemon" "Tor hidden service running"
-		forever $ do
-			(conn, _) <- S.accept soc
-			h <- setupHandle conn
+		serveUnixSocket sock $ \conn -> do
 			ok <- atomically $ ifM (isFullTBMQueue q)
 				( return False
 				, do
-					writeTBMQueue q h
+					writeTBMQueue q conn
 					return True
 				)
 			unless ok $ do
-				hClose h
+				hClose conn
 				warningIO "dropped Tor connection, too busy"
 	go _ Nothing = debugM "remotedaemon" "Tor hidden service not enabled"
 

From e08691b393d0ea7db332146010ee18fae2708fe2 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 24 Dec 2016 12:49:28 -0400
Subject: [PATCH 361/367] enable-tor: When run as a regular user, test a
 connection back to the hidden service over tor.

This way we know that after enable-tor, the tor hidden service is fully
published and working, and so there should be no problems with it at
pairing time.

It has to start up its own temporary listener on the hidden service. It
would be nice to have it start the remotedaemon running, so that extra
step is not needed afterwards. But, there may already be a remotedaemon
running, in communication with the assistant and we don't want to start
another one. I thought about trying to HUP any running remotedaemon, but
Windows does not make it easy to do that. In any case, having the user
start the remotedaemon themselves lets them know it needs to be running
to serve the hidden service.

This commit was sponsored by Boyd Stephen Smith Jr. on Patreon.
---
 CHANGELOG                                   |  2 +
 Command/EnableTor.hs                        | 77 +++++++++++++++++++--
 P2P/IO.hs                                   |  2 +-
 doc/tips/peer_to_peer_network_with_tor.mdwn |  3 +-
 doc/todo/tor.mdwn                           |  1 -
 5 files changed, 78 insertions(+), 7 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index 220aeea41a..2d7ea22a70 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -22,6 +22,8 @@ git-annex (6.20161211) UNRELEASED; urgency=medium
   * enable-tor: Put tor sockets in /var/lib/tor-annex/, rather
     than in /etc/tor/hidden_service/.
   * enable-tor: No longer needs to be run as root.
+  * enable-tor: When run as a regular user, test a connection back to 
+    the hidden service over tor.
   * Fix build with directory-1.3.
   * Debian: Suggest tor and magic-wormhole.
   * Debian: Build webapp on armel.
diff --git a/Command/EnableTor.hs b/Command/EnableTor.hs
index c81968a559..6f145413d0 100644
--- a/Command/EnableTor.hs
+++ b/Command/EnableTor.hs
@@ -10,18 +10,22 @@
 module Command.EnableTor where
 
 import Command
+import qualified Annex
 import P2P.Address
 import Utility.Tor
 import Annex.UUID
 import Config.Files
+import P2P.IO
+import qualified P2P.Protocol as P2P
+import Utility.ThreadScheduler
 
+import Control.Concurrent.Async
+import qualified Network.Socket as S
 #ifndef mingw32_HOST_OS
 import Utility.Su
 import System.Posix.User
 #endif
 
--- This runs as root, so avoid making any commits or initializing
--- git-annex, or doing other things that create root-owned files.
 cmd :: Command
 cmd = noCommit $ dontCheck repoExists $
 	command "enable-tor" SectionSetup "enable tor hidden service"
@@ -30,6 +34,8 @@ cmd = noCommit $ dontCheck repoExists $
 seek :: CmdParams -> CommandSeek
 seek = withWords start
 
+-- This runs as root, so avoid making any commits or initializing
+-- git-annex, or doing other things that create root-owned files.
 start :: [String] -> CommandStart
 start os = do
 	uuid <- getUUID
@@ -42,11 +48,12 @@ start os = do
 			Nothing -> giveup "Need user-id parameter."
 			Just userid -> go uuid userid
 		else do
-			liftIO $ putStrLn "Need root access to enable tor..."
+			showStart "enable-tor" ""
+			showLongNote "Need root access to enable tor..."
 			gitannex <- liftIO readProgramFile
 			let ps = [Param (cmdname cmd), Param (show curruserid)]
 			ifM (liftIO $ runAsRoot gitannex ps)
-				( stop
+				( next $ next checkHiddenService
 				, giveup $ unwords $
 					[ "Failed to run as root:" , gitannex ] ++ toCommand ps
 				)
@@ -59,3 +66,65 @@ start os = do
 			addHiddenService torAppName userid (fromUUID uuid)
 		storeP2PAddress $ TorAnnex onionaddr onionport
 		stop
+
+checkHiddenService :: CommandCleanup
+checkHiddenService = bracket setup cleanup go
+  where
+	setup = do
+		showLongNote "Tor hidden service is configured. Checking connection to it. This may take a few minutes."
+		startlistener
+	
+	cleanup = liftIO . cancel
+	
+	go _ = check (150 :: Int) =<< filter istoraddr <$> loadP2PAddresses
+	
+	istoraddr (TorAnnex _ _) = True
+
+	check 0 _ = giveup "Still unable to connect to hidden service. It might not yet be usable by others. Please check Tor's logs for details."
+	check _ [] = giveup "Somehow didn't get an onion address."
+	check n addrs@(addr:_) = do
+		g <- Annex.gitRepo
+		-- Connect but don't bother trying to auth,
+		-- we just want to know if the tor circuit works.
+		cv <- liftIO $ tryNonAsync $ connectPeer g addr
+		case cv of
+			Left e -> do
+				warning $ "Unable to connect to hidden service. It may not yet have propigated to the Tor network. (" ++ show e ++ ") Will retry.."
+				liftIO $ threadDelaySeconds (Seconds 2)
+				check (n-1) addrs
+			Right conn -> do
+				liftIO $ closeConnection conn
+				showLongNote "Tor hidden service is working."
+				return True
+	
+	-- Unless the remotedaemon is already listening on the hidden
+	-- service's socket, start a listener. This is only run during the
+	-- check, and it refuses all auth attempts.
+	startlistener = do
+		r <- Annex.gitRepo
+		u <- getUUID
+		uid <- liftIO getRealUserID
+		let ident = fromUUID u
+		v <- liftIO $ getHiddenServiceSocketFile torAppName uid ident
+		case v of
+			Just sockfile -> ifM (liftIO $ haslistener sockfile)
+				( liftIO $ async $ return ()
+				, liftIO $ async $ runlistener sockfile u r
+				)
+			Nothing -> giveup "Could not find socket file in Tor configuration!"
+	
+	runlistener sockfile u r = serveUnixSocket sockfile $ \h -> do
+		let conn = P2PConnection
+			{ connRepo = r
+			, connCheckAuth = const False
+			, connIhdl = h
+			, connOhdl = h
+			}
+		void $ runNetProto conn $ P2P.serveAuth u
+		hClose h
+
+	haslistener sockfile = catchBoolIO $ do
+		soc <- S.socket S.AF_UNIX S.Stream S.defaultProtocol
+		S.connect soc (S.SockAddrUnix sockfile)
+		S.close soc
+		return True
diff --git a/P2P/IO.hs b/P2P/IO.hs
index 89f712fcae..ee1724d7b3 100644
--- a/P2P/IO.hs
+++ b/P2P/IO.hs
@@ -72,7 +72,7 @@ closeConnection conn = do
 -- closing the Handle when done.
 --
 -- Note that while the callback is running, other connections won't be
--- processes, so longterm work should be run in a separate thread by
+-- processed, so longterm work should be run in a separate thread by
 -- the callback.
 serveUnixSocket :: FilePath -> (Handle -> IO ()) -> IO ()
 serveUnixSocket unixsocket serveconn = do
diff --git a/doc/tips/peer_to_peer_network_with_tor.mdwn b/doc/tips/peer_to_peer_network_with_tor.mdwn
index ce00b0424f..0fdc346251 100644
--- a/doc/tips/peer_to_peer_network_with_tor.mdwn
+++ b/doc/tips/peer_to_peer_network_with_tor.mdwn
@@ -26,7 +26,8 @@ In each git-annex repository, run these commands:
 	git annex enable-tor
 	git annex remotedaemon
 
-Now git-annex is running as a Tor hidden service, but 
+The enable-tor command may prompt for the root password, since it
+configures Tor. Now git-annex is running as a Tor hidden service, but 
 it will only talk to peers after pairing with them.
 
 In both repositories, run this command:
diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index f0c1936776..cb0bc4d410 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -4,7 +4,6 @@ Mostly working!
 
 Current todo list:
 
-* Make enable-tor check connection back to itself to verify tor is working.
 * When a transfer can't be done because another transfer of the same 
   object is already in progress, the message about this is output by the
   remotedaemon --debug, but not forwarded to the peer, which shows

From 993f30ac927da6ddb632a6ca42f4ca495d6c1bfa Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 24 Dec 2016 12:53:57 -0400
Subject: [PATCH 362/367] update

---
 doc/todo/tor.mdwn | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index cb0bc4d410..3a4a190c4d 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -4,6 +4,7 @@ Mostly working!
 
 Current todo list:
 
+* Webapp UI to set it up.
 * When a transfer can't be done because another transfer of the same 
   object is already in progress, the message about this is output by the
   remotedaemon --debug, but not forwarded to the peer, which shows
@@ -16,8 +17,6 @@ Eventually:
 * Limiting authtokens to read-only access.
 * Revoking authtokens. (This and read-only need a name associated with an
   authtoken, so the user can adjust its configuration after creating it.)
-* Pairing via magic wormhole.
-* Webapp UI to set it up.
 * friend-of-a-friend peer discovery to build more interconnected networks
   of nodes
 * Discovery of nodes on same LAN, and direct connection to them.

From f7e81245c4a296eb1edf55c758e23bc7663b3d3f Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 24 Dec 2016 12:58:45 -0400
Subject: [PATCH 363/367] comment

---
 ...mment_6_0cada5a6154438c674f01d449378ffe9._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/bugs/cannot_change_locale___40__en__95__US.UTF-8__41__/comment_6_0cada5a6154438c674f01d449378ffe9._comment

diff --git a/doc/bugs/cannot_change_locale___40__en__95__US.UTF-8__41__/comment_6_0cada5a6154438c674f01d449378ffe9._comment b/doc/bugs/cannot_change_locale___40__en__95__US.UTF-8__41__/comment_6_0cada5a6154438c674f01d449378ffe9._comment
new file mode 100644
index 0000000000..1942a6f522
--- /dev/null
+++ b/doc/bugs/cannot_change_locale___40__en__95__US.UTF-8__41__/comment_6_0cada5a6154438c674f01d449378ffe9._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2016-12-24T16:57:45Z"
+ content="""
+This is an old closed bug report. The recent comments are about a
+completely unrelated issue, which I suspect was fixed by
+[[!commit 95c8b37544c383983712c5c368dd803c51bf8eeb]].
+
+Please file new bug reports if you have an issue, if the old bug report was
+closed years ago.
+"""]]

From 0e04b22491bd618242f52931c8786a476d7fa970 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 24 Dec 2016 13:08:07 -0400
Subject: [PATCH 364/367] fix su params

---
 Utility/Su.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Utility/Su.hs b/Utility/Su.hs
index 001a09fbc6..97dc789d44 100644
--- a/Utility/Su.hs
+++ b/Utility/Su.hs
@@ -45,7 +45,7 @@ runAsRoot cmd ps = go =<< firstM (inPath . fst) =<< selectcmds
 	
 	-- These will only work when run in a console.
 	consolecmds = 
-		[ ("su", [Param "-c", Param "--", Param cmd] ++ ps)
+		[ ("su", [Param "-c", Param cmd, Param "--"] ++ ps)
 		, ("sudo", [Param cmd] ++ ps)
 		, ("su-to-root", [Param "-c", Param shellcmd])
 		]

From d6b401c4584ca21c32cdb0ba0d445b7592789f85 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 24 Dec 2016 13:10:51 -0400
Subject: [PATCH 365/367] todo

---
 doc/todo/tor.mdwn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/todo/tor.mdwn b/doc/todo/tor.mdwn
index 3a4a190c4d..734670839d 100644
--- a/doc/todo/tor.mdwn
+++ b/doc/todo/tor.mdwn
@@ -14,6 +14,7 @@ Current todo list:
 
 Eventually:
 
+* Windows and Android support.
 * Limiting authtokens to read-only access.
 * Revoking authtokens. (This and read-only need a name associated with an
   authtoken, so the user can adjust its configuration after creating it.)

From c89a9e6ca5ffa2d79fe643b4423b79d98a407d9d Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 24 Dec 2016 13:23:52 -0400
Subject: [PATCH 366/367] really fix su command

---
 Utility/Su.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Utility/Su.hs b/Utility/Su.hs
index 97dc789d44..44a95c39f8 100644
--- a/Utility/Su.hs
+++ b/Utility/Su.hs
@@ -45,7 +45,7 @@ runAsRoot cmd ps = go =<< firstM (inPath . fst) =<< selectcmds
 	
 	-- These will only work when run in a console.
 	consolecmds = 
-		[ ("su", [Param "-c", Param cmd, Param "--"] ++ ps)
+		[ ("su", [Param "-c", Param shellcmd])
 		, ("sudo", [Param cmd] ++ ps)
 		, ("su-to-root", [Param "-c", Param shellcmd])
 		]

From 8484c0c1976b2daa4be674b09f6f54c15c746aa6 Mon Sep 17 00:00:00 2001
From: Joey Hess <joeyh@joeyh.name>
Date: Sat, 24 Dec 2016 14:46:31 -0400
Subject: [PATCH 367/367] Always use filesystem encoding for all file and
 handle reads and writes.

This is a big scary change. I have convinced myself it should be safe. I
hope!
---
 Annex/Branch.hs               |  2 +-
 Annex/CatFile.hs              |  1 +
 Annex/Content/Direct.hs       |  6 ++---
 Annex/DirHashes.hs            |  1 +
 Annex/Journal.hs              |  3 +--
 Annex/Link.hs                 |  2 +-
 Annex/Ssh.hs                  |  1 +
 Annex/VariantFile.hs          |  1 +
 Assistant/TransferrerPool.hs  |  2 --
 Assistant/WebApp/Control.hs   |  2 +-
 Backend/Utilities.hs          |  1 +
 Build/DistributionUpdate.hs   |  2 ++
 Build/EvilSplicer.hs          |  5 +++--
 CHANGELOG                     |  2 ++
 CmdLine/Batch.hs              |  4 +---
 Command/AddUrl.hs             |  1 +
 Command/ImportFeed.hs         |  2 +-
 Command/P2P.hs                |  3 +--
 Command/TransferKeys.hs       |  5 +----
 Command/Vicfg.hs              |  6 ++---
 Common.hs                     |  1 -
 Config.hs                     |  2 +-
 Database/Handle.hs            |  2 +-
 Git/CatFile.hs                |  1 +
 Git/Command.hs                |  6 +----
 Git/Config.hs                 |  5 -----
 Git/HashObject.hs             |  1 -
 Git/Queue.hs                  |  1 -
 Git/Repair.hs                 |  2 +-
 Git/UnionMerge.hs             |  4 ++--
 Git/UpdateIndex.hs            |  1 -
 Logs/Transfer.hs              |  5 ++---
 Logs/Unused.hs                |  4 ++--
 Messages.hs                   |  1 -
 P2P/IO.hs                     |  1 -
 Remote/BitTorrent.hs          |  1 +
 Remote/External.hs            |  3 ---
 Test.hs                       |  4 +++-
 Utility/CoProcess.hs          |  6 ++---
 Utility/ExternalSHA.hs        |  2 --
 Utility/FileSystemEncoding.hs | 41 +++++++++++++++++++----------------
 Utility/Lsof.hs               |  5 ++---
 Utility/MagicWormhole.hs      |  4 +---
 Utility/Misc.hs               | 17 ---------------
 Utility/Quvi.hs               |  3 ---
 Utility/Shell.hs              |  5 ++---
 git-annex.hs                  |  2 ++
 git-union-merge.hs            |  2 ++
 48 files changed, 75 insertions(+), 109 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 9663311d51..c90958ab05 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -61,6 +61,7 @@ import qualified Annex.Queue
 import Annex.Branch.Transitions
 import qualified Annex
 import Annex.Hook
+import Utility.FileSystemEncoding
 
 {- Name of the branch that is used to store git-annex's information. -}
 name :: Git.Ref
@@ -436,7 +437,6 @@ stageJournal jl = withIndex $ do
 	g <- gitRepo
 	let dir = gitAnnexJournalDir g
 	(jlogf, jlogh) <- openjlog
-	liftIO $ fileEncoding jlogh
 	h <- hashObjectHandle
 	withJournalHandle $ \jh ->
 		Git.UpdateIndex.streamUpdateIndex g
diff --git a/Annex/CatFile.hs b/Annex/CatFile.hs
index b1d8fba287..25952dfecd 100644
--- a/Annex/CatFile.hs
+++ b/Annex/CatFile.hs
@@ -33,6 +33,7 @@ import Git.FilePath
 import Git.Index
 import qualified Git.Ref
 import Annex.Link
+import Utility.FileSystemEncoding
 
 catFile :: Git.Branch -> FilePath -> Annex L.ByteString
 catFile branch file = do
diff --git a/Annex/Content/Direct.hs b/Annex/Content/Direct.hs
index 2007360e31..734a0c1b94 100644
--- a/Annex/Content/Direct.hs
+++ b/Annex/Content/Direct.hs
@@ -52,8 +52,7 @@ associatedFiles key = do
 associatedFilesRelative :: Key -> Annex [FilePath] 
 associatedFilesRelative key = do
 	mapping <- calcRepo $ gitAnnexMapping key
-	liftIO $ catchDefaultIO [] $ withFile mapping ReadMode $ \h -> do
-		fileEncoding h
+	liftIO $ catchDefaultIO [] $ withFile mapping ReadMode $ \h ->
 		-- Read strictly to ensure the file is closed
 		-- before changeAssociatedFiles tries to write to it.
 		-- (Especially needed on Windows.)
@@ -68,8 +67,7 @@ changeAssociatedFiles key transform = do
 	let files' = transform files
 	when (files /= files') $
 		modifyContent mapping $
-			liftIO $ viaTmp writeFileAnyEncoding mapping $
-				unlines files'
+			liftIO $ viaTmp writeFile mapping $ unlines files'
 	top <- fromRepo Git.repoPath
 	return $ map (top </>) files'
 
diff --git a/Annex/DirHashes.hs b/Annex/DirHashes.hs
index 004536ca79..ed20cfb8a7 100644
--- a/Annex/DirHashes.hs
+++ b/Annex/DirHashes.hs
@@ -26,6 +26,7 @@ import Common
 import Types.Key
 import Types.GitConfig
 import Types.Difference
+import Utility.FileSystemEncoding
 
 type Hasher = Key -> FilePath
 
diff --git a/Annex/Journal.hs b/Annex/Journal.hs
index e4faa48653..184bb0ab04 100644
--- a/Annex/Journal.hs
+++ b/Annex/Journal.hs
@@ -37,7 +37,6 @@ setJournalFile _jl file content = do
 	let tmpfile = tmp </> takeFileName jfile
 	liftIO $ do
 		withFile tmpfile WriteMode $ \h -> do
-			fileEncoding h
 #ifdef mingw32_HOST_OS
 			hSetNewlineMode h noNewlineTranslation
 #endif
@@ -53,7 +52,7 @@ getJournalFile _jl = getJournalFileStale
  - changes. -}
 getJournalFileStale :: FilePath -> Annex (Maybe String)
 getJournalFileStale file = inRepo $ \g -> catchMaybeIO $
-	readFileStrictAnyEncoding $ journalFile file g
+	readFileStrict $ journalFile file g
 
 {- List of files that have updated content in the journal. -}
 getJournalledFiles :: JournalLocked -> Annex [FilePath]
diff --git a/Annex/Link.hs b/Annex/Link.hs
index 90312a04a0..fcc300beec 100644
--- a/Annex/Link.hs
+++ b/Annex/Link.hs
@@ -24,6 +24,7 @@ import Git.Types
 import Git.FilePath
 import Annex.HashObject
 import Utility.FileMode
+import Utility.FileSystemEncoding
 
 import qualified Data.ByteString.Lazy as L
 
@@ -63,7 +64,6 @@ getAnnexLinkTarget' file coresymlinks = if coresymlinks
 			Nothing -> fallback
 
 	probefilecontent f = withFile f ReadMode $ \h -> do
-		fileEncoding h
 		-- The first 8k is more than enough to read; link
 		-- files are small.
 		s <- take 8192 <$> hGetContents h
diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs
index 4f879436b5..512f0375c2 100644
--- a/Annex/Ssh.hs
+++ b/Annex/Ssh.hs
@@ -33,6 +33,7 @@ import qualified Git.Url
 import Config
 import Annex.Path
 import Utility.Env
+import Utility.FileSystemEncoding
 import Types.CleanupActions
 import Git.Env
 #ifndef mingw32_HOST_OS
diff --git a/Annex/VariantFile.hs b/Annex/VariantFile.hs
index 9bf027b5ce..17658a9c61 100644
--- a/Annex/VariantFile.hs
+++ b/Annex/VariantFile.hs
@@ -8,6 +8,7 @@
 module Annex.VariantFile where
 
 import Annex.Common
+import Utility.FileSystemEncoding
 
 import Data.Hash.MD5
 
diff --git a/Assistant/TransferrerPool.hs b/Assistant/TransferrerPool.hs
index 7c0cb44151..892e156e8b 100644
--- a/Assistant/TransferrerPool.hs
+++ b/Assistant/TransferrerPool.hs
@@ -74,8 +74,6 @@ mkTransferrer program batchmaker = do
 		, std_in = CreatePipe
 		, std_out = CreatePipe
 		}
-	fileEncoding readh
-	fileEncoding writeh
 	return $ Transferrer
 		{ transferrerRead = readh
 		, transferrerWrite = writeh
diff --git a/Assistant/WebApp/Control.hs b/Assistant/WebApp/Control.hs
index 93b923fed9..8a0e82119d 100644
--- a/Assistant/WebApp/Control.hs
+++ b/Assistant/WebApp/Control.hs
@@ -74,5 +74,5 @@ getLogR :: Handler Html
 getLogR = page "Logs" Nothing $ do
 	logfile <- liftAnnex $ fromRepo gitAnnexLogFile
 	logs <- liftIO $ listLogs logfile
-	logcontent <- liftIO $ concat <$> mapM readFileStrictAnyEncoding logs
+	logcontent <- liftIO $ concat <$> mapM readFile logs
 	$(widgetFile "control/log")
diff --git a/Backend/Utilities.hs b/Backend/Utilities.hs
index 04221650b4..d1fb94f2af 100644
--- a/Backend/Utilities.hs
+++ b/Backend/Utilities.hs
@@ -10,6 +10,7 @@ module Backend.Utilities where
 import Data.Hash.MD5
 
 import Annex.Common
+import Utility.FileSystemEncoding
 
 {- Generates a keyName from an input string. Takes care of sanitizing it.
  - If it's not too long, the full string is used as the keyName.
diff --git a/Build/DistributionUpdate.hs b/Build/DistributionUpdate.hs
index 814927e995..dd18a78837 100644
--- a/Build/DistributionUpdate.hs
+++ b/Build/DistributionUpdate.hs
@@ -14,6 +14,7 @@ import Build.Version (getChangelogVersion, Version)
 import Utility.UserInfo
 import Utility.Url
 import Utility.Tmp
+import Utility.FileSystemEncoding
 import qualified Git.Construct
 import qualified Annex
 import Annex.Content
@@ -50,6 +51,7 @@ autobuilds =
 
 main :: IO ()
 main = do
+	useFileSystemEncoding
 	version <- liftIO getChangelogVersion
 	repodir <- getRepoDir
 	changeWorkingDirectory repodir
diff --git a/Build/EvilSplicer.hs b/Build/EvilSplicer.hs
index ca690c2501..32d9a1c9fb 100644
--- a/Build/EvilSplicer.hs
+++ b/Build/EvilSplicer.hs
@@ -210,7 +210,6 @@ applySplices destdir imports splices@(first:_) = do
 	when (oldcontent /= Just newcontent) $ do
 		putStrLn $ "splicing " ++ f
 		withFile dest WriteMode $ \h -> do
-		        fileEncoding h
 			hPutStr h newcontent
 		        hClose h
   where
@@ -721,7 +720,9 @@ parsecAndReplace p s = case parse find "" s of
 	find = many $ try (Right <$> p) <|> (Left <$> anyChar)
 
 main :: IO ()
-main = go =<< getArgs
+main = do
+	useFileSystemEncoding
+	go =<< getArgs
   where
 	go (destdir:log:header:[]) = run destdir log (Just header)
 	go (destdir:log:[]) = run destdir log Nothing
diff --git a/CHANGELOG b/CHANGELOG
index 2d7ea22a70..7a0ca2eb20 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -24,6 +24,8 @@ git-annex (6.20161211) UNRELEASED; urgency=medium
   * enable-tor: No longer needs to be run as root.
   * enable-tor: When run as a regular user, test a connection back to 
     the hidden service over tor.
+  * Always use filesystem encoding for all file and handle reads and
+    writes.
   * Fix build with directory-1.3.
   * Debian: Suggest tor and magic-wormhole.
   * Debian: Build webapp on armel.
diff --git a/CmdLine/Batch.hs b/CmdLine/Batch.hs
index 6ef21372f5..82038314c0 100644
--- a/CmdLine/Batch.hs
+++ b/CmdLine/Batch.hs
@@ -57,9 +57,7 @@ batchInput parser a = go =<< batchLines
 	parseerr s = giveup $ "Batch input parse failure: " ++ s
 
 batchLines :: Annex [String]
-batchLines = liftIO $ do
-	fileEncoding stdin
-	lines <$> getContents
+batchLines = liftIO $ lines <$> getContents
 
 -- Runs a CommandStart in batch mode.
 --
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index e49d2727c7..8cc1484406 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -27,6 +27,7 @@ import Types.UrlContents
 import Annex.FileMatcher
 import Logs.Location
 import Utility.Metered
+import Utility.FileSystemEncoding
 import qualified Annex.Transfer as Transfer
 import Annex.Quvi
 import qualified Utility.Quvi as Quvi
diff --git a/Command/ImportFeed.hs b/Command/ImportFeed.hs
index 832ec1246f..ea936e84a5 100644
--- a/Command/ImportFeed.hs
+++ b/Command/ImportFeed.hs
@@ -156,7 +156,7 @@ downloadFeed url
 		liftIO $ withTmpFile "feed" $ \f h -> do
 			hClose h
 			ifM (Url.download url f uo)
-				( parseFeedString <$> readFileStrictAnyEncoding f
+				( parseFeedString <$> readFileStrict f
 				, return Nothing
 				)
 
diff --git a/Command/P2P.hs b/Command/P2P.hs
index afa5f9dc68..4ba3e43d53 100644
--- a/Command/P2P.hs
+++ b/Command/P2P.hs
@@ -161,7 +161,6 @@ performPairing remotename addrs = do
 	getcode ourcode = do
 		putStr "Enter the other repository's pairing code: "
 		hFlush stdout
-		fileEncoding stdin
 		l <- getLine
 		case Wormhole.toCode l of
 			Just code
@@ -236,7 +235,7 @@ wormholePairing remotename ouraddrs ui = do
 				then return ReceiveFailed
 				else do
 					r <- liftIO $ tryIO $
-						readFileStrictAnyEncoding recvf
+						readFileStrict recvf
 					case r of
 						Left _e -> return ReceiveFailed
 						Right s -> maybe 
diff --git a/Command/TransferKeys.hs b/Command/TransferKeys.hs
index 2ac7845890..d875f496d0 100644
--- a/Command/TransferKeys.hs
+++ b/Command/TransferKeys.hs
@@ -56,10 +56,7 @@ runRequests
 	-> (TransferRequest -> Annex Bool)
 	-> Annex ()
 runRequests readh writeh a = do
-	liftIO $ do
-		hSetBuffering readh NoBuffering
-		fileEncoding readh
-		fileEncoding writeh
+	liftIO $ hSetBuffering readh NoBuffering
 	go =<< readrequests
   where
 	go (d:rn:k:f:rest) = do
diff --git a/Command/Vicfg.hs b/Command/Vicfg.hs
index 64daa598b7..d9e8b88232 100644
--- a/Command/Vicfg.hs
+++ b/Command/Vicfg.hs
@@ -41,7 +41,7 @@ start = do
 	createAnnexDirectory $ parentDir f
 	cfg <- getCfg
 	descs <- uuidDescriptions
-	liftIO $ writeFileAnyEncoding f $ genCfg cfg descs
+	liftIO $ writeFile f $ genCfg cfg descs
 	vicfg cfg f
 	stop
 
@@ -51,11 +51,11 @@ vicfg curcfg f = do
 	-- Allow EDITOR to be processed by the shell, so it can contain options.
 	unlessM (liftIO $ boolSystem "sh" [Param "-c", Param $ unwords [vi, shellEscape f]]) $
 		giveup $ vi ++ " exited nonzero; aborting"
-	r <- parseCfg (defCfg curcfg) <$> liftIO (readFileStrictAnyEncoding f)
+	r <- parseCfg (defCfg curcfg) <$> liftIO (readFileStrict f)
 	liftIO $ nukeFile f
 	case r of
 		Left s -> do
-			liftIO $ writeFileAnyEncoding f s
+			liftIO $ writeFile f s
 			vicfg curcfg f
 		Right newcfg -> setCfg curcfg newcfg
 
diff --git a/Common.hs b/Common.hs
index 5213863b9f..2e28117b6c 100644
--- a/Common.hs
+++ b/Common.hs
@@ -29,7 +29,6 @@ import Utility.Directory as X
 import Utility.Monad as X
 import Utility.Data as X
 import Utility.Applicative as X
-import Utility.FileSystemEncoding as X
 import Utility.PosixFiles as X hiding (fileSize)
 import Utility.FileSize as X
 import Utility.Network as X
diff --git a/Config.hs b/Config.hs
index be60852daa..84736cac35 100644
--- a/Config.hs
+++ b/Config.hs
@@ -112,7 +112,7 @@ configureSmudgeFilter = do
 		createDirectoryIfMissing True (takeDirectory lf)
 		writeFile lf (lfs ++ "\n" ++ stdattr)
   where
-	readattr = liftIO . catchDefaultIO "" . readFileStrictAnyEncoding
+	readattr = liftIO . catchDefaultIO "" . readFileStrict
 	stdattr = unlines
 		[ "* filter=annex"
 		, ".* !filter"
diff --git a/Database/Handle.hs b/Database/Handle.hs
index 748feaa976..9071cd5380 100644
--- a/Database/Handle.hs
+++ b/Database/Handle.hs
@@ -69,7 +69,7 @@ openDb db tablename = do
 	worker <- async (workerThread (T.pack db) tablename jobs)
 	
 	-- work around https://github.com/yesodweb/persistent/issues/474
-	liftIO setConsoleEncoding
+	liftIO useFileSystemEncoding
 
 	return $ DbHandle worker jobs
 
diff --git a/Git/CatFile.hs b/Git/CatFile.hs
index 061349f054..4935cdffa0 100644
--- a/Git/CatFile.hs
+++ b/Git/CatFile.hs
@@ -37,6 +37,7 @@ import Git.Command
 import Git.Types
 import Git.FilePath
 import qualified Utility.CoProcess as CoProcess
+import Utility.FileSystemEncoding
 
 data CatFileHandle = CatFileHandle 
 	{ catFileProcess :: CoProcess.CoProcessHandle
diff --git a/Git/Command.hs b/Git/Command.hs
index 2060563688..adea7622e0 100644
--- a/Git/Command.hs
+++ b/Git/Command.hs
@@ -53,7 +53,6 @@ runQuiet params repo = withQuietOutput createProcessSuccess $
 pipeReadLazy :: [CommandParam] -> Repo -> IO (String, IO Bool)
 pipeReadLazy params repo = assertLocal repo $ do
 	(_, Just h, _, pid) <- createProcess p { std_out = CreatePipe }
-	fileEncoding h
 	c <- hGetContents h
 	return (c, checkSuccessProcess pid)
   where
@@ -66,7 +65,6 @@ pipeReadLazy params repo = assertLocal repo $ do
 pipeReadStrict :: [CommandParam] -> Repo -> IO String
 pipeReadStrict params repo = assertLocal repo $
 	withHandle StdoutHandle (createProcessChecked ignoreFailureProcess) p $ \h -> do
-		fileEncoding h
 		output <- hGetContentsStrict h
 		hClose h
 		return output
@@ -81,9 +79,7 @@ pipeWriteRead params writer repo = assertLocal repo $
 	writeReadProcessEnv "git" (toCommand $ gitCommandLine params repo) 
 		(gitEnv repo) writer (Just adjusthandle)
   where
-	adjusthandle h = do
-		fileEncoding h
-		hSetNewlineMode h noNewlineTranslation
+	adjusthandle h = hSetNewlineMode h noNewlineTranslation
 
 {- Runs a git command, feeding it input on a handle with an action. -}
 pipeWrite :: [CommandParam] -> Repo -> (Handle -> IO ()) -> IO ()
diff --git a/Git/Config.hs b/Git/Config.hs
index 3d6239560f..65bd9b7ba3 100644
--- a/Git/Config.hs
+++ b/Git/Config.hs
@@ -79,10 +79,6 @@ global = do
 {- Reads git config from a handle and populates a repo with it. -}
 hRead :: Repo -> Handle -> IO Repo
 hRead repo h = do
-	-- We use the FileSystemEncoding when reading from git-config,
-	-- because it can contain arbitrary filepaths (and other strings)
-	-- in any encoding.
-	fileEncoding h
 	val <- hGetContentsStrict h
 	store val repo
 
@@ -167,7 +163,6 @@ coreBare = "core.bare"
 fromPipe :: Repo -> String -> [CommandParam] -> IO (Either SomeException (Repo, String))
 fromPipe r cmd params = try $
 	withHandle StdoutHandle createProcessSuccess p $ \h -> do
-		fileEncoding h
 		val <- hGetContentsStrict h
 		r' <- store val r
 		return (r', val)
diff --git a/Git/HashObject.hs b/Git/HashObject.hs
index 4cd54ef54c..399e36d460 100644
--- a/Git/HashObject.hs
+++ b/Git/HashObject.hs
@@ -41,7 +41,6 @@ hashFile h file = CoProcess.query h send receive
  - interface does not allow batch hashing without using temp files. -}
 hashBlob :: HashObjectHandle -> String -> IO Sha
 hashBlob h s = withTmpFile "hash" $ \tmp tmph -> do
-	fileEncoding tmph
 #ifdef mingw32_HOST_OS
 	hSetNewlineMode tmph noNewlineTranslation
 #endif
diff --git a/Git/Queue.hs b/Git/Queue.hs
index 0b0025b0a7..ee1f83ca9c 100644
--- a/Git/Queue.hs
+++ b/Git/Queue.hs
@@ -159,7 +159,6 @@ runAction repo action@(CommandAction {}) = do
 #ifndef mingw32_HOST_OS
 	let p = (proc "xargs" $ "-0":"git":toCommand gitparams) { env = gitEnv repo }
 	withHandle StdinHandle createProcessSuccess p $ \h -> do
-		fileEncoding h
 		hPutStr h $ intercalate "\0" $ toCommand $ getFiles action
 		hClose h
 #else
diff --git a/Git/Repair.hs b/Git/Repair.hs
index fcfc03600c..1baf51a644 100644
--- a/Git/Repair.hs
+++ b/Git/Repair.hs
@@ -614,4 +614,4 @@ successfulRepair = fst
 safeReadFile :: FilePath -> IO String
 safeReadFile f = do
 	allowRead f
-	readFileStrictAnyEncoding f
+	readFileStrict f
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 9ae8295ae2..c6157a9ed0 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -22,6 +22,7 @@ import Git.UpdateIndex
 import Git.HashObject
 import Git.Types
 import Git.FilePath
+import Utility.FileSystemEncoding
 
 {- Performs a union merge between two branches, staging it in the index.
  - Any previously staged changes in the index will be lost.
@@ -94,8 +95,7 @@ mergeFile info file hashhandle h = case filter (/= nullSha) [Ref asha, Ref bsha]
 	-- We don't know how the file is encoded, but need to
 	-- split it into lines to union merge. Using the
 	-- FileSystemEncoding for this is a hack, but ensures there
-	-- are no decoding errors. Note that this works because
-	-- hashObject sets fileEncoding on its write handle.
+	-- are no decoding errors.
 	getcontents s = lines . encodeW8NUL . L.unpack <$> catObject h s
 
 {- Calculates a union merge between a list of refs, with contents.
diff --git a/Git/UpdateIndex.hs b/Git/UpdateIndex.hs
index 55c5b3bb21..7fdc9450f4 100644
--- a/Git/UpdateIndex.hs
+++ b/Git/UpdateIndex.hs
@@ -55,7 +55,6 @@ startUpdateIndex :: Repo -> IO UpdateIndexHandle
 startUpdateIndex repo = do
 	(Just h, _, _, p) <- createProcess (gitCreateProcess params repo)
 		{ std_in = CreatePipe }
-	fileEncoding h
 	return $ UpdateIndexHandle p h
   where
 	params = map Param ["update-index", "-z", "--index-info"]
diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs
index 65a4e3796d..28f7b0a263 100644
--- a/Logs/Transfer.hs
+++ b/Logs/Transfer.hs
@@ -220,8 +220,7 @@ parseTransferFile file
 	bits = splitDirectories file
 
 writeTransferInfoFile :: TransferInfo -> FilePath -> IO ()
-writeTransferInfoFile info tfile = writeFileAnyEncoding tfile $
-	writeTransferInfo info
+writeTransferInfoFile info tfile = writeFile tfile $ writeTransferInfo info
 
 {- File format is a header line containing the startedTime and any
  - bytesComplete value. Followed by a newline and the associatedFile.
@@ -243,7 +242,7 @@ writeTransferInfo info = unlines
 
 readTransferInfoFile :: Maybe PID -> FilePath -> IO (Maybe TransferInfo)
 readTransferInfoFile mpid tfile = catchDefaultIO Nothing $
-	readTransferInfo mpid <$> readFileStrictAnyEncoding tfile
+	readTransferInfo mpid <$> readFileStrict tfile
 
 readTransferInfo :: Maybe PID -> String -> Maybe TransferInfo
 readTransferInfo mpid s = TransferInfo
diff --git a/Logs/Unused.hs b/Logs/Unused.hs
index 1035d1246c..2361fedbcf 100644
--- a/Logs/Unused.hs
+++ b/Logs/Unused.hs
@@ -66,7 +66,7 @@ updateUnusedLog prefix m = do
 writeUnusedLog :: FilePath -> UnusedLog -> Annex ()
 writeUnusedLog prefix l = do
 	logfile <- fromRepo $ gitAnnexUnusedLog prefix
-	liftIO $ viaTmp writeFileAnyEncoding logfile $ unlines $ map format $ M.toList l
+	liftIO $ viaTmp writeFile logfile $ unlines $ map format $ M.toList l
   where
 	format (k, (i, Just t)) = show i ++ " " ++ key2file k ++ " " ++ show t
 	format (k, (i, Nothing)) = show i ++ " " ++ key2file k
@@ -76,7 +76,7 @@ readUnusedLog prefix = do
 	f <- fromRepo $ gitAnnexUnusedLog prefix
 	ifM (liftIO $ doesFileExist f)
 		( M.fromList . mapMaybe parse . lines
-			<$> liftIO (readFileStrictAnyEncoding f)
+			<$> liftIO (readFileStrict f)
 		, return M.empty
 		)
   where
diff --git a/Messages.hs b/Messages.hs
index 0ab1f72bbb..0036e57596 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -183,7 +183,6 @@ setupConsole = do
 		<$> streamHandler stderr DEBUG
 		<*> pure preciseLogFormatter
 	updateGlobalLogger rootLoggerName (setLevel NOTICE . setHandlers [s])
-	setConsoleEncoding
 	{- Force output to be line buffered. This is normally the case when
 	 - it's connected to a terminal, but may not be when redirected to
 	 - a file or a pipe. -}
diff --git a/P2P/IO.hs b/P2P/IO.hs
index ee1724d7b3..9ebb102f19 100644
--- a/P2P/IO.hs
+++ b/P2P/IO.hs
@@ -99,7 +99,6 @@ setupHandle s = do
 	h <- socketToHandle s ReadWriteMode
 	hSetBuffering h LineBuffering
 	hSetBinaryMode h False
-	fileEncoding h
 	return h
 
 -- Purposefully incomplete interpreter of Proto.
diff --git a/Remote/BitTorrent.hs b/Remote/BitTorrent.hs
index 899c57e3eb..0ec78aa642 100644
--- a/Remote/BitTorrent.hs
+++ b/Remote/BitTorrent.hs
@@ -21,6 +21,7 @@ import Types.CleanupActions
 import Messages.Progress
 import Utility.Metered
 import Utility.Tmp
+import Utility.FileSystemEncoding
 import Backend.URL
 import Annex.Perms
 import Annex.UUID
diff --git a/Remote/External.hs b/Remote/External.hs
index 0b0e1dc18b..7091a657c8 100644
--- a/Remote/External.hs
+++ b/Remote/External.hs
@@ -384,9 +384,6 @@ startExternal external = do
 		p <- propgit g basep
 		(Just hin, Just hout, Just herr, ph) <- 
 			createProcess p `catchIO` runerr
-		fileEncoding hin
-		fileEncoding hout
-		fileEncoding herr
 		stderrelay <- async $ errrelayer herr
 		checkearlytermination =<< getProcessExitCode ph
 		cv <- newTVarIO $ externalDefaultConfig external
diff --git a/Test.hs b/Test.hs
index 3f67277211..0ab7bf1309 100644
--- a/Test.hs
+++ b/Test.hs
@@ -95,6 +95,7 @@ import qualified Utility.HumanTime
 import qualified Utility.ThreadScheduler
 import qualified Utility.Base64
 import qualified Utility.Tmp
+import qualified Utility.FileSystemEncoding
 import qualified Command.Uninit
 import qualified CmdLine.GitAnnex as GitAnnex
 #ifndef mingw32_HOST_OS
@@ -1675,7 +1676,8 @@ test_add_subdirs = intmpclonerepo $ do
 	 - calculated correctly for files in subdirs. -}
 	unlessM (unlockedFiles <$> getTestMode) $ do
 		git_annex "sync" [] @? "sync failed"
-		l <- annexeval $ decodeBS <$> Annex.CatFile.catObject (Git.Types.Ref "HEAD:dir/foo")
+		l <- annexeval $ Utility.FileSystemEncoding.decodeBS
+			<$> Annex.CatFile.catObject (Git.Types.Ref "HEAD:dir/foo")
 		"../.git/annex/" `isPrefixOf` l @? ("symlink from subdir to .git/annex is wrong: " ++ l)
 
 	createDirectory "dir2"
diff --git a/Utility/CoProcess.hs b/Utility/CoProcess.hs
index 94d5ac3bc4..2bae40fbae 100644
--- a/Utility/CoProcess.hs
+++ b/Utility/CoProcess.hs
@@ -47,10 +47,10 @@ start' s = do
 	rawMode to
 	return $ CoProcessState pid to from s
   where
-	rawMode h = do
-		fileEncoding h
 #ifdef mingw32_HOST_OS
-		hSetNewlineMode h noNewlineTranslation
+	rawMode h = hSetNewlineMode h noNewlineTranslation
+#else
+	rawMode _ = return ()
 #endif
 
 stop :: CoProcessHandle -> IO ()
diff --git a/Utility/ExternalSHA.hs b/Utility/ExternalSHA.hs
index e581697aef..7b08820040 100644
--- a/Utility/ExternalSHA.hs
+++ b/Utility/ExternalSHA.hs
@@ -14,7 +14,6 @@ module Utility.ExternalSHA (externalSHA) where
 
 import Utility.SafeCommand
 import Utility.Process
-import Utility.FileSystemEncoding
 import Utility.Misc
 import Utility.Exception
 
@@ -30,7 +29,6 @@ externalSHA command shasize file = do
 		Left _ -> Left (command ++ " failed")
   where
 	readsha args = withHandle StdoutHandle createProcessSuccess p $ \h -> do
-		fileEncoding h
 		output  <- hGetContentsStrict h
 		hClose h
 		return output
diff --git a/Utility/FileSystemEncoding.hs b/Utility/FileSystemEncoding.hs
index eab98337a8..be43ace95e 100644
--- a/Utility/FileSystemEncoding.hs
+++ b/Utility/FileSystemEncoding.hs
@@ -1,6 +1,6 @@
 {- GHC File system encoding handling.
  -
- - Copyright 2012-2014 Joey Hess <id@joeyh.name>
+ - Copyright 2012-2016 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}
@@ -9,7 +9,7 @@
 {-# OPTIONS_GHC -fno-warn-tabs #-}
 
 module Utility.FileSystemEncoding (
-	fileEncoding,
+	useFileSystemEncoding,
 	withFilePath,
 	md5FilePath,
 	decodeBS,
@@ -19,7 +19,6 @@ module Utility.FileSystemEncoding (
 	encodeW8NUL,
 	decodeW8NUL,
 	truncateFilePath,
-	setConsoleEncoding,
 ) where
 
 import qualified GHC.Foreign as GHC
@@ -39,19 +38,30 @@ import qualified Data.ByteString.Lazy.UTF8 as L8
 
 import Utility.Exception
 
-{- Sets a Handle to use the filesystem encoding. This causes data
- - written or read from it to be encoded/decoded the same
- - as ghc 7.4 does to filenames etc. This special encoding
- - allows "arbitrary undecodable bytes to be round-tripped through it".
+{- Makes all subsequent Handles that are opened, as well as stdio Handles,
+ - use the filesystem encoding, instead of the encoding of the current
+ - locale.
+ -
+ - The filesystem encoding allows "arbitrary undecodable bytes to be
+ - round-tripped through it". This avoids encoded failures when data is not
+ - encoded matching the current locale.
+ -
+ - Note that code can still use hSetEncoding to change the encoding of a
+ - Handle. This only affects the default encoding.
  -}
-fileEncoding :: Handle -> IO ()
+useFileSystemEncoding :: IO ()
+useFileSystemEncoding = do
 #ifndef mingw32_HOST_OS
-fileEncoding h = hSetEncoding h =<< Encoding.getFileSystemEncoding
+	e <- Encoding.getFileSystemEncoding
 #else
-{- The file system encoding does not work well on Windows,
- - and Windows only has utf FilePaths anyway. -}
-fileEncoding h = hSetEncoding h Encoding.utf8
+	{- The file system encoding does not work well on Windows,
+	 - and Windows only has utf FilePaths anyway. -}
+	let e = Encoding.utf8
 #endif
+	hSetEncoding stdin e
+	hSetEncoding stdout e
+	hSetEncoding stderr e
+	Encoding.setLocaleEncoding e	
 
 {- Marshal a Haskell FilePath into a NUL terminated C string using temporary
  - storage. The FilePath is encoded using the filesystem encoding,
@@ -165,10 +175,3 @@ truncateFilePath n = reverse . go [] n . L8.fromString
 					else go (c:coll) (cnt - x') (L8.drop 1 bs)
 			_ -> coll
 #endif
-
-{- This avoids ghc's output layer crashing on invalid encoded characters in
- - filenames when printing them out. -}
-setConsoleEncoding :: IO ()
-setConsoleEncoding = do
-	fileEncoding stdout
-	fileEncoding stderr
diff --git a/Utility/Lsof.hs b/Utility/Lsof.hs
index 433b7c6799..27d34b5925 100644
--- a/Utility/Lsof.hs
+++ b/Utility/Lsof.hs
@@ -47,9 +47,8 @@ queryDir path = query ["+d", path]
  -}
 query :: [String] -> IO [(FilePath, LsofOpenMode, ProcessInfo)]
 query opts =
-	withHandle StdoutHandle (createProcessChecked checkSuccessProcess) p $ \h -> do
-		fileEncoding h
-		parse <$> hGetContentsStrict h
+	withHandle StdoutHandle (createProcessChecked checkSuccessProcess) p $
+		parse <$$> hGetContentsStrict
   where
 	p = proc "lsof" ("-F0can" : opts)
 
diff --git a/Utility/MagicWormhole.hs b/Utility/MagicWormhole.hs
index 9a99cba33f..e217dcdca6 100644
--- a/Utility/MagicWormhole.hs
+++ b/Utility/MagicWormhole.hs
@@ -27,7 +27,6 @@ import Utility.Process
 import Utility.SafeCommand
 import Utility.Monad
 import Utility.Misc
-import Utility.FileSystemEncoding
 import Utility.Env
 import Utility.Path
 
@@ -105,8 +104,7 @@ sendFile f (CodeObserver observer) ps = do
 	-- Work around stupid stdout buffering behavior of python.
 	-- See https://github.com/warner/magic-wormhole/issues/108
 	environ <- addEntry "PYTHONUNBUFFERED" "1" <$> getEnvironment
-	runWormHoleProcess p { env = Just environ} $ \_hin hout -> do
-		fileEncoding hout
+	runWormHoleProcess p { env = Just environ} $ \_hin hout ->
 		findcode =<< words <$> hGetContents hout
   where
 	p = wormHoleProcess (Param "send" : ps ++ [File f])
diff --git a/Utility/Misc.hs b/Utility/Misc.hs
index ebb42576b2..4498c0a03e 100644
--- a/Utility/Misc.hs
+++ b/Utility/Misc.hs
@@ -10,9 +10,6 @@
 
 module Utility.Misc where
 
-import Utility.FileSystemEncoding
-import Utility.Monad
-
 import System.IO
 import Control.Monad
 import Foreign
@@ -35,20 +32,6 @@ hGetContentsStrict = hGetContents >=> \s -> length s `seq` return s
 readFileStrict :: FilePath -> IO String
 readFileStrict = readFile >=> \s -> length s `seq` return s
 
-{-  Reads a file strictly, and using the FileSystemEncoding, so it will
- -  never crash on a badly encoded file. -}
-readFileStrictAnyEncoding :: FilePath -> IO String
-readFileStrictAnyEncoding f = withFile f ReadMode $ \h -> do
-	fileEncoding h
-	hClose h `after` hGetContentsStrict h
-
-{- Writes a file, using the FileSystemEncoding so it will never crash
- - on a badly encoded content string. -}
-writeFileAnyEncoding :: FilePath -> String -> IO ()
-writeFileAnyEncoding f content = withFile f WriteMode $ \h -> do
-	fileEncoding h
-	hPutStr h content
-
 {- Like break, but the item matching the condition is not included
  - in the second result list.
  -
diff --git a/Utility/Quvi.hs b/Utility/Quvi.hs
index 417ab7041c..d33d79bb8e 100644
--- a/Utility/Quvi.hs
+++ b/Utility/Quvi.hs
@@ -153,11 +153,8 @@ httponly :: QuviParams
 httponly Quvi04 = [Param "-c", Param "http"]
 httponly _ = [] -- No way to do it with 0.9?
 
-{- Both versions of quvi will output utf-8 encoded data even when
- - the locale doesn't support it. -}
 readQuvi :: [String] -> IO String
 readQuvi ps = withHandle StdoutHandle createProcessSuccess p $ \h -> do
-	fileEncoding h
 	r <- hGetContentsStrict h
 	hClose h
 	return r
diff --git a/Utility/Shell.hs b/Utility/Shell.hs
index 860ee11dda..7adb651289 100644
--- a/Utility/Shell.hs
+++ b/Utility/Shell.hs
@@ -48,9 +48,8 @@ findShellCommand f = do
 #ifndef mingw32_HOST_OS
 	defcmd
 #else
-	l <- catchDefaultIO Nothing $ withFile f ReadMode $ \h -> do
-		fileEncoding h
-		headMaybe . lines <$> hGetContents h
+	l <- catchDefaultIO Nothing $ withFile f ReadMode $
+		headMaybe . lines <$$> hGetContents h
 	case l of
 		Just ('#':'!':rest) -> case words rest of
 			[] -> defcmd
diff --git a/git-annex.hs b/git-annex.hs
index d5fab7f477..e30d320b92 100644
--- a/git-annex.hs
+++ b/git-annex.hs
@@ -15,6 +15,7 @@ import qualified CmdLine.GitAnnex
 import qualified CmdLine.GitAnnexShell
 import qualified CmdLine.GitRemoteTorAnnex
 import qualified Test
+import Utility.FileSystemEncoding
 
 #ifdef mingw32_HOST_OS
 import Utility.UserInfo
@@ -23,6 +24,7 @@ import Utility.Env
 
 main :: IO ()
 main = withSocketsDo $ do
+	useFileSystemEncoding
 	ps <- getArgs
 #ifdef mingw32_HOST_OS
 	winEnv
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 3bf628c754..18c88b1a9e 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -14,6 +14,7 @@ import qualified Git.CurrentRepo
 import qualified Git.Branch
 import qualified Git.Index
 import qualified Git
+import Utility.FileSystemEncoding
 
 header :: String
 header = "Usage: git-union-merge ref ref newref"
@@ -39,6 +40,7 @@ parseArgs = do
 
 main :: IO ()
 main = do
+	useFileSystemEncoding
 	[aref, bref, newref] <- map Git.Ref <$> parseArgs
 	g <- Git.Config.read =<< Git.CurrentRepo.get
 	_ <- Git.Index.override (tmpIndex g) g