From d8f21b7955c48f121924d187b1be07f53edbff93 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Jul 2011 23:54:36 -0400 Subject: [PATCH 001/111] add news item for git-annex 3.20110719 --- doc/news/version_0.20110610.mdwn | 6 ------ doc/news/version_3.20110719.mdwn | 7 +++++++ 2 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 doc/news/version_0.20110610.mdwn create mode 100644 doc/news/version_3.20110719.mdwn diff --git a/doc/news/version_0.20110610.mdwn b/doc/news/version_0.20110610.mdwn deleted file mode 100644 index 9ab9e09076..0000000000 --- a/doc/news/version_0.20110610.mdwn +++ /dev/null @@ -1,6 +0,0 @@ -git-annex 0.20110610 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Add --numcopies option. - * Add --trust, --untrust, and --semitrust options. - * get --from is the same as copy --from - * Bugfix: Fix fsck to not think all SHAnE keys are bad."""]] \ No newline at end of file diff --git a/doc/news/version_3.20110719.mdwn b/doc/news/version_3.20110719.mdwn new file mode 100644 index 0000000000..5beae32270 --- /dev/null +++ b/doc/news/version_3.20110719.mdwn @@ -0,0 +1,7 @@ +git-annex 3.20110719 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * add: Be even more robust to avoid ever leaving the file seemingly deleted. + Closes: #[634233](http://bugs.debian.org/634233) + * Bugfix: Make add ../ work. + * Support the standard git -c name=value + * unannex: Clean up use of git commit -a."""]] \ No newline at end of file From 1d88b966e29e0b064a142a712d6a350b3f19e2d1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 24 Jul 2011 15:15:43 +0200 Subject: [PATCH 002/111] fix broken img --- doc/index.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/index.mdwn b/doc/index.mdwn index 8975c82de7..4b7159cd53 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -12,7 +12,7 @@ To get a feel for it, see the [[walkthrough]]. * [[forum]] * [[comments]] * [[contact]] -* Flattr this +* Flattr this [[News]]: From 50edbb03eb0582dd568a6fb83fcdb410828c1ea1 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkvSZ1AFJdY_1FeutZr_KWeqtzjZta1PNE" Date: Thu, 28 Jul 2011 17:40:53 +0000 Subject: [PATCH 003/111] --- ...es_which_have_names_containing_spaces.mdwn | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 doc/bugs/rsync_special_remote_fails_to___96__get__96___files_which_have_names_containing_spaces.mdwn diff --git a/doc/bugs/rsync_special_remote_fails_to___96__get__96___files_which_have_names_containing_spaces.mdwn b/doc/bugs/rsync_special_remote_fails_to___96__get__96___files_which_have_names_containing_spaces.mdwn new file mode 100644 index 0000000000..464c06b442 --- /dev/null +++ b/doc/bugs/rsync_special_remote_fails_to___96__get__96___files_which_have_names_containing_spaces.mdwn @@ -0,0 +1,48 @@ + ~$ mkdir test annex + ~$ cd test + ~$ git init + Initialized empty Git repository in /home/user/test/.git/ + ~$ git annex init test + init test ok + ~$ git annex initremote localrsync encryption=none type=rsync rsyncurl=localhost:annex/ + initremote localrsync ok + ~$ cp /home/user/Music/Charming\ Hostess/Eat/03\ Mi\ Nuera.ogg ./ + ~$ git annex add 03\ Mi\ Nuera.ogg + add 03 Mi Nuera.ogg ok + (Recording state in git...) + ~$ git commit -m "add ogg" + fatal: No HEAD commit to compare with (yet) + fatal: No HEAD commit to compare with (yet) + [master (root-commit) 12608af] add ogg + 1 files changed, 1 insertions(+), 0 deletions(-) + create mode 120000 03 Mi Nuera.ogg + ~$ git annex move 03\ Mi\ Nuera.ogg --to localrsync + move 03 Mi Nuera.ogg (checking localrsync...) (to localrsync...) + sending incremental file list + 1X/ + 1X/39/ + 1X/39/WORM-s6296772-m1311874383--03 Mi Nuera.ogg/ + 1X/39/WORM-s6296772-m1311874383--03 Mi Nuera.ogg/WORM-s6296772-m1311874383--03 Mi Nuera.ogg + 6296772 100% 42.98MB/s 0:00:00 (xfer#1, to-check=0/5) + + sent 6297754 bytes received 43 bytes 4198531.33 bytes/sec + total size is 6296772 speedup is 1.00 + ok + ~$ git annex get 03\ Mi\ Nuera.ogg + get 03 Mi Nuera.ogg (from localrsync...) + rsync: link_stat "/home/user/annex/1X/39/WORM-s6296772-m1311874383--03" failed: No such file or directory (2) + rsync: link_stat "/home/user/Mi" failed: No such file or directory (2) + rsync: change_dir "/home/user/Nuera.ogg" failed: No such file or directory (2) + rsync: link_stat "/home/user/Mi" failed: No such file or directory (2) + rsync: link_stat "/home/user/Nuera.ogg" failed: No such file or directory (2) + + sent 8 bytes received 12 bytes 13.33 bytes/sec + total size is 0 speedup is 0.00 + rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1526) [Receiver=3.0.7] + + rsync failed -- run git annex again to resume file transfer + Unable to access these remotes: localrsync + Try making some of these repositories available: + b8b1ea7a-b93f-11e0-b712-d7bffb6e61e6 -- localrsync + failed + git-annex: 1 failed From 45bbf210a1210172c7c7b87879ed74f7c8ccbdba Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Jul 2011 15:28:21 +0200 Subject: [PATCH 004/111] Fix shell escaping in rsync special remote. --- Remote/Rsync.hs | 10 ++++++---- debian/changelog | 6 ++++++ ..._96___files_which_have_names_containing_spaces.mdwn | 2 ++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs index f073e7bd79..9535376097 100644 --- a/Remote/Rsync.hs +++ b/Remote/Rsync.hs @@ -93,10 +93,13 @@ rsyncSetup u c = do return c' rsyncKey :: RsyncOpts -> Key -> String -rsyncKey o k = rsyncUrl o hashDirMixed k f f +rsyncKey o k = rsyncUrl o hashDirMixed k shellEscape (f f) where f = keyFile k +rsyncKeyDir :: RsyncOpts -> Key -> String +rsyncKeyDir o k = rsyncUrl o hashDirMixed k shellEscape (keyFile k) + store :: RsyncOpts -> Key -> Annex Bool store o k = do g <- Annex.gitRepo @@ -136,7 +139,7 @@ remove o k = withRsyncScratchDir $ \tmp -> do [ Params "--delete --recursive" , partialParams , Param $ addTrailingPathSeparator dummy - , Param $ parentDir $ rsyncKey o k + , Param $ rsyncKeyDir o k ] checkPresent :: Git.Repo -> RsyncOpts -> Key -> Annex (Either IOException Bool) @@ -147,8 +150,7 @@ checkPresent r o k = do res <- liftIO $ boolSystem "sh" [Param "-c", Param cmd] return $ Right res where - cmd = "rsync --quiet " ++ testfile ++ " 2>/dev/null" - testfile = shellEscape $ rsyncKey o k + cmd = "rsync --quiet " ++ shellEscape (rsyncKey o k) ++ " 2>/dev/null" {- Rsync params to enable resumes of sending files safely, - ensure that files are only moved into place once complete diff --git a/debian/changelog b/debian/changelog index fab4927cdd..748a945d7b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (3.20110720) UNRELEASED; urgency=low + + * Fix shell escaping in rsync special remote. + + -- Joey Hess Fri, 29 Jul 2011 15:27:30 +0200 + git-annex (3.20110719) unstable; urgency=low * add: Be even more robust to avoid ever leaving the file seemingly deleted. diff --git a/doc/bugs/rsync_special_remote_fails_to___96__get__96___files_which_have_names_containing_spaces.mdwn b/doc/bugs/rsync_special_remote_fails_to___96__get__96___files_which_have_names_containing_spaces.mdwn index 464c06b442..040d86bb87 100644 --- a/doc/bugs/rsync_special_remote_fails_to___96__get__96___files_which_have_names_containing_spaces.mdwn +++ b/doc/bugs/rsync_special_remote_fails_to___96__get__96___files_which_have_names_containing_spaces.mdwn @@ -46,3 +46,5 @@ b8b1ea7a-b93f-11e0-b712-d7bffb6e61e6 -- localrsync failed git-annex: 1 failed + +> [[fixed|done]] --[[Joey]] From 16c55f61f2d4f060f20094335fdfcac827f7d508 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmL8pteP2jbYJUn1M3CbeLDvz2SWAA1wtg" Date: Sat, 30 Jul 2011 17:28:42 +0000 Subject: [PATCH 005/111] --- doc/install/OSX.mdwn | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn index 23cb1b62e2..2d376d053f 100644 --- a/doc/install/OSX.mdwn +++ b/doc/install/OSX.mdwn @@ -1,5 +1,7 @@ +Install Haskel Platform from [[http://hackage.haskell.org/platform/mac.html]]. The version provided by Macports is too old to work with current versions of git-annex. Then execute + 
-sudo port install haskell-platform git-core ossp-uuid md5sha1sum coreutils pcre
+sudo port install git-core ossp-uuid md5sha1sum coreutils pcre
 sudo cabal update
 sudo cabal install missingh
 sudo cabal install utf8-string
@@ -21,7 +23,7 @@ make
 sudo make install
-Originally posted by Jon at --[[Joey]] +Originally posted by Jon at --[[Joey]], modified by [[kristianrumberg]] See also: From d9cdb5eb457c5069d20c0738158de2423770b3e4 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmL8pteP2jbYJUn1M3CbeLDvz2SWAA1wtg" Date: Sat, 30 Jul 2011 18:04:25 +0000 Subject: [PATCH 006/111] --- doc/install/OSX.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn index 2d376d053f..983ac3a2fd 100644 --- a/doc/install/OSX.mdwn +++ b/doc/install/OSX.mdwn @@ -2,6 +2,8 @@ Install Haskel Platform from [[http://hackage.haskell.org/platform/mac.html]]. T 
 sudo port install git-core ossp-uuid md5sha1sum coreutils pcre
+
+sudo ln -s /opt/local/include/pcre.h  /usr/include/pcre.h # This is hack that allows pcre-light to find pcre
 sudo cabal update
 sudo cabal install missingh
 sudo cabal install utf8-string

From ad4528cce83709eb075158ef15fb1fd6faeb1171 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sun, 31 Jul 2011 01:20:35 +0000
Subject: [PATCH 007/111]

---
 doc/bugs/Prevent_accidental_merges.mdwn | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/bugs/Prevent_accidental_merges.mdwn

diff --git a/doc/bugs/Prevent_accidental_merges.mdwn b/doc/bugs/Prevent_accidental_merges.mdwn
new file mode 100644
index 0000000000..9b0c8a3de6
--- /dev/null
+++ b/doc/bugs/Prevent_accidental_merges.mdwn
@@ -0,0 +1,9 @@
+With the storage layout v3, pulling the git-annex branch into the master branch is... less than ideal.
+
+The fact that the two branches contain totally different data make an accidental merge worse, arguably.
+
+Adding a tiny binary file called .gitnomerge to both branches would solve that without any noticeable overhead.
+
+Yes, there is an argument to be made that this is too much hand-holding, but I still think it's worth it.
+
+-- Richard

From 344f2c3ec18baaefe6de14a32fc1209459470bf5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmL8pteP2jbYJUn1M3CbeLDvz2SWAA1wtg"
 
Date: Sun, 31 Jul 2011 15:24:29 +0000
Subject: [PATCH 008/111] Added a comment: Solution

---
 ...nt_2_f5ebb7f43dcef861ecc13373fb1e263f._comment | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_2_f5ebb7f43dcef861ecc13373fb1e263f._comment

diff --git a/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_2_f5ebb7f43dcef861ecc13373fb1e263f._comment b/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_2_f5ebb7f43dcef861ecc13373fb1e263f._comment
new file mode 100644
index 0000000000..9601003798
--- /dev/null
+++ b/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_2_f5ebb7f43dcef861ecc13373fb1e263f._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmL8pteP2jbYJUn1M3CbeLDvz2SWAA1wtg"
+ nickname="Kristian"
+ subject="Solution"
+ date="2011-07-31T15:24:25Z"
+ content="""
+Yes, it can read id3-tags and guess titles from movie filenames but it sometimes gets confused by the filename metadata provided by the WORM-backend.
+
+I think I have a good enough solution to this problem. It's not efficient when it comes to renames but handles adding and deletion just fine
+
+    rsync -vaL --delete source dest
+
+The -L flag looks at symbolic links and copies the actual data they are pointing to. Of course \"source\" must have all data locally for this to work.
+
+"""]]

From b48fec3846ac3e32b473499c007c98865236848c Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmFgsNxmnGznb5bbmcoWhoQOoxZZ-io61s"
 
Date: Mon, 1 Aug 2011 09:52:22 +0000
Subject: [PATCH 009/111]

---
 doc/bugs/Cabal_dependency_monadIO_missing.mdwn | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Cabal_dependency_monadIO_missing.mdwn

diff --git a/doc/bugs/Cabal_dependency_monadIO_missing.mdwn b/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
new file mode 100644
index 0000000000..03d31ea34f
--- /dev/null
+++ b/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
@@ -0,0 +1,8 @@
+Just issuing the command `cabal install` results in the following error message.
+
+    Command/Add.hs:54:3:
+        No instance for (Control.Monad.IO.Control.MonadControlIO
+                           (Control.Monad.State.Lazy.StateT Annex.AnnexState IO))
+          arising from a use of `handle' at Command/Add.hs:54:3-24
+
+Adding the dependency for `monadIO` to `git-annex.cabal` should fix this?

From a746623a334fa1673179621fe5c9c9d6d407851f Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmFgsNxmnGznb5bbmcoWhoQOoxZZ-io61s"
 
Date: Mon, 1 Aug 2011 09:54:51 +0000
Subject: [PATCH 010/111]

---
 doc/bugs/Cabal_dependency_monadIO_missing.mdwn | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/Cabal_dependency_monadIO_missing.mdwn b/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
index 03d31ea34f..cf4b138128 100644
--- a/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
+++ b/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
@@ -5,4 +5,5 @@ Just issuing the command `cabal install` results in the following error message.
                            (Control.Monad.State.Lazy.StateT Annex.AnnexState IO))
           arising from a use of `handle' at Command/Add.hs:54:3-24
 
-Adding the dependency for `monadIO` to `git-annex.cabal` should fix this?
+Adding the dependency for `monadIO` to `git-annex.cabal` should fix this?  
+-- Thomas

From d3f6f4fe34e812fafbc487c969e6aaa0d49de33f Mon Sep 17 00:00:00 2001
From: "http://christian.amsuess.com/chrysn" 
Date: Thu, 4 Aug 2011 15:25:21 +0000
Subject: [PATCH 011/111] fixed internal link

---
 doc/special_remotes/web.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/special_remotes/web.mdwn b/doc/special_remotes/web.mdwn
index 68df31ef4d..6991ce4e44 100644
--- a/doc/special_remotes/web.mdwn
+++ b/doc/special_remotes/web.mdwn
@@ -1,5 +1,5 @@
 git-annex can use the WWW as a special remote, downloading urls to files.
-See [[walkthrough/using_web_web]] for usage examples.
+See [[walkthrough/using_the_web]] for usage examples.
 
 ## notes
 

From 578e66996446f3c29f431fc6da0ce9a525c15135 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 5 Aug 2011 10:27:22 -0400
Subject: [PATCH 012/111] response

---
 doc/bugs/Cabal_dependency_monadIO_missing.mdwn | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/bugs/Cabal_dependency_monadIO_missing.mdwn b/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
index cf4b138128..b5213b8aa5 100644
--- a/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
+++ b/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
@@ -7,3 +7,8 @@ Just issuing the command `cabal install` results in the following error message.
 
 Adding the dependency for `monadIO` to `git-annex.cabal` should fix this?  
 -- Thomas
+
+> No, it's already satisfied by `monad-control` being listed as a
+> dependency in the cabal file. Your system might be old/new/or broken,
+> perhaps it's time to provide some details about the version of haskell
+> and of `monad-control` you have installed? --[[Joey]] 

From 46ddb4e66aa1ba2a569eb8734c8ca0f40f3930d4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 5 Aug 2011 10:29:21 -0400
Subject: [PATCH 013/111] response

---
 doc/bugs/Prevent_accidental_merges.mdwn | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/bugs/Prevent_accidental_merges.mdwn b/doc/bugs/Prevent_accidental_merges.mdwn
index 9b0c8a3de6..3e30e02235 100644
--- a/doc/bugs/Prevent_accidental_merges.mdwn
+++ b/doc/bugs/Prevent_accidental_merges.mdwn
@@ -7,3 +7,8 @@ Adding a tiny binary file called .gitnomerge to both branches would solve that w
 Yes, there is an argument to be made that this is too much hand-holding, but I still think it's worth it.
 
 -- Richard
+
+> It should be as easy to undo such an accidential merge
+> as it is to undo any other git commit, right? I quite like that git-annex 
+> no longer adds any clutter to the master branch, and would be reluctant
+> to change that. --[[Joey]]

From d2492f990fdfa13a04429edaa5373a39e75cc9f0 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawliqfHEW134uawIUPwyKiyOdoF-oI5TxnQ"
 
Date: Fri, 5 Aug 2011 22:35:48 +0000
Subject: [PATCH 014/111]

---
 doc/cheatsheet.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/cheatsheet.mdwn b/doc/cheatsheet.mdwn
index 9f7c146c82..9ccb22e3e2 100644
--- a/doc/cheatsheet.mdwn
+++ b/doc/cheatsheet.mdwn
@@ -1,4 +1,4 @@
-A suppliment to the [[walkthrough]].
+A supplement to the [[walkthrough]].
 
 [[!toc]]
 

From 3ffc0bb4f57e20350f59c0e331656e54877916aa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 6 Aug 2011 12:50:20 -0400
Subject: [PATCH 015/111] foo

---
 Backend.hs     | 2 +-
 Backend/SHA.hs | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index 3429e8f42c..6942692e83 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -65,7 +65,7 @@ orderedList = do
 genKey :: FilePath -> Maybe (Backend Annex) -> Annex (Maybe (Key, Backend Annex))
 genKey file trybackend = do
 	bs <- orderedList
-	let bs' = maybe bs (:bs) trybackend
+	let bs' = maybe bs (: bs) trybackend
 	genKey' bs' file
 genKey' :: [Backend Annex] -> FilePath -> Annex (Maybe (Key, Backend Annex))
 genKey' [] _ = return Nothing
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index c1d7136485..bae19be003 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -32,7 +32,7 @@ sizes :: [Int]
 sizes = [1, 256, 512, 224, 384]
 
 backends :: [Backend Annex]
--- order is slightly significant; want sha1 first ,and more general
+-- order is slightly significant; want sha1 first, and more general
 -- sizes earlier
 backends = catMaybes $ map genBackend sizes ++ map genBackendE sizes
 
@@ -107,7 +107,7 @@ keyValueE size file = keyValue size file >>= maybe (return Nothing) addE
 				then "" -- probably not really an extension
 				else naiveextension
 
--- A key's checksum is checked during fsck.
+{- A key's checksum is checked during fsck. -}
 checkKeyChecksum :: SHASize -> Key -> Annex Bool
 checkKeyChecksum size key = do
 	g <- Annex.gitRepo

From dd8e649f49212c46df23f329299c64e13a3c90e1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 6 Aug 2011 14:45:58 -0400
Subject: [PATCH 016/111] fix file name for web remote log files

The key name was not being sufficiently escaped, although it didn't break
anything due to luck. Switch to properly escaped key names for the log
filename, with a fallback to the buggy old name.
---
 Remote/Web.hs | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/Remote/Web.hs b/Remote/Web.hs
index 2f8fac23b5..cd028a06d9 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -67,10 +67,17 @@ gen r _ _ =
 {- The urls for a key are stored in remote/web/hash/key.log 
  - in the git-annex branch. -}
 urlLog :: Key -> FilePath
-urlLog key = "remote/web"  hashDirLower key  show key ++ ".log"
+urlLog key = "remote/web"  hashDirLower key  keyFile key ++ ".log"
+oldurlLog :: Key -> FilePath
+{- A bug used to store the urls elsewhere. -}
+oldurlLog key = "remote/web"  hashDirLower key  show key ++ ".log"
 
 getUrls :: Key -> Annex [URLString]
-getUrls key = currentLog (urlLog key)
+getUrls key = do
+	us <- currentLog (urlLog key)
+	if null us
+		then currentLog (oldurlLog key)
+		else return us
 
 {- Records a change in an url for a key. -}
 setUrl :: Key -> URLString -> LogStatus -> Annex ()

From dede05171bc9431778da72e5e1235c69db9fa38e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 6 Aug 2011 14:57:22 -0400
Subject: [PATCH 017/111] addurl: --fast can be used to avoid immediately
 downloading the url.

The tricky part about this is that to generate a key, the file must be
present already. Worked around by adding (back) an URL key type, which
is used for addurl --fast.
---
 Backend.hs         |  3 ++-
 Backend/URL.hs     | 28 ++++++++++++++++++++++++++++
 Command/Add.hs     | 21 ++++++++++++---------
 Command/AddUrl.hs  | 27 +++++++++++++++++++--------
 Command/Migrate.hs |  2 +-
 debian/changelog   |  1 +
 doc/git-annex.mdwn |  2 ++
 7 files changed, 65 insertions(+), 19 deletions(-)
 create mode 100644 Backend/URL.hs

diff --git a/Backend.hs b/Backend.hs
index 6942692e83..0bb9f4b575 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -32,9 +32,10 @@ import Messages
 -- When adding a new backend, import it here and add it to the list.
 import qualified Backend.WORM
 import qualified Backend.SHA
+import qualified Backend.URL
 
 list :: [Backend Annex]
-list = Backend.WORM.backends ++ Backend.SHA.backends
+list = Backend.WORM.backends ++ Backend.SHA.backends ++ Backend.URL.backends
 
 {- List of backends in the order to try them when storing a new key. -}
 orderedList :: Annex [Backend Annex]
diff --git a/Backend/URL.hs b/Backend/URL.hs
new file mode 100644
index 0000000000..f20aa1f95e
--- /dev/null
+++ b/Backend/URL.hs
@@ -0,0 +1,28 @@
+{- git-annex "URL" backend -- keys whose content is available from urls.
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Backend.URL (
+	backends,
+	fromUrl
+) where
+
+import Types.Backend
+import Types.Key
+import Types
+
+backends :: [Backend Annex]
+backends = [backend]
+
+backend :: Backend Annex
+backend = Types.Backend.Backend {
+	name = "URL",
+	getKey = const (return Nothing),
+	fsckKey = const (return True)
+}
+
+fromUrl :: String -> Key
+fromUrl url = stubKey { keyName = url, keyBackendName = "URL" }
diff --git a/Command/Add.hs b/Command/Add.hs
index 58c0143dd0..d8947fb07c 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -8,6 +8,7 @@
 module Command.Add where
 
 import Control.Monad.State (liftIO)
+import Control.Monad (when)
 import System.Posix.Files
 import System.Directory
 import Control.Exception.Control (handle)
@@ -52,7 +53,7 @@ perform (file, backend) = do
 		Nothing -> stop
 		Just (key, _) -> do
 			handle (undo file key) $ moveAnnex key file
-			next $ cleanup file key
+			next $ cleanup file key True
 
 {- On error, put the file back so it doesn't seem to have vanished.
  - This can be called before or after the symlink is in place. -}
@@ -72,18 +73,20 @@ undo file key e = do
 			g <- Annex.gitRepo
 			liftIO $ renameFile (gitAnnexLocation g key) file
 
-cleanup :: FilePath -> Key -> CommandCleanup
-cleanup file key = do
+cleanup :: FilePath -> Key -> Bool -> CommandCleanup
+cleanup file key hascontent = do
 	handle (undo file key) $ do
 		link <- calcGitLink file key
 		liftIO $ createSymbolicLink link file
-		logStatus key InfoPresent
+
+		when hascontent $ do
+			logStatus key InfoPresent
 	
-		-- touch the symlink to have the same mtime as the
-		-- file it points to
-		s <- liftIO $ getFileStatus file
-		let mtime = modificationTime s
-		liftIO $ touch file (TimeSpec mtime) False
+			-- touch the symlink to have the same mtime as the
+			-- file it points to
+			s <- liftIO $ getFileStatus file
+			let mtime = modificationTime s
+			liftIO $ touch file (TimeSpec mtime) False
 
 	force <- Annex.getState Annex.force
 	if force
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 1b12362e9f..e87de384b3 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -17,10 +17,10 @@ import qualified Backend
 import qualified Remote.Web
 import qualified Command.Add
 import qualified Annex
+import qualified Backend.URL
 import Messages
 import Content
 import PresenceLog
-import Types.Key
 import Locations
 import Utility
 
@@ -42,9 +42,14 @@ start s = do
 			
 perform :: String -> FilePath -> CommandPerform
 perform url file = do
+	fast <- Annex.getState Annex.fast
+	if fast then nodownload url file else download url file
+
+download :: String -> FilePath -> CommandPerform
+download url file = do
 	g <- Annex.gitRepo
 	showAction $ "downloading " ++ url ++ " "
-	let dummykey = stubKey { keyName = url, keyBackendName = "URL" }
+	let dummykey = Backend.URL.fromUrl url
 	let tmp = gitAnnexTmpLocation g dummykey
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
 	ok <- Remote.Web.download [url] tmp
@@ -57,9 +62,16 @@ perform url file = do
 				Just (key, _) -> do
 					moveAnnex key tmp
 					Remote.Web.setUrl key url InfoPresent
-					next $ Command.Add.cleanup file key
+					next $ Command.Add.cleanup file key True
 		else stop
 
+nodownload :: String -> FilePath -> CommandPerform
+nodownload url file = do
+	let key = Backend.URL.fromUrl url
+	Remote.Web.setUrl key url InfoPresent
+	
+	next $ Command.Add.cleanup file key False
+
 url2file :: URI -> IO FilePath
 url2file url = do
 	let parts = filter safe $ split "/" $ uriPath url
@@ -75,8 +87,7 @@ url2file url = do
 			e <- doesFileExist file
 			when e $ error "already have this url"
 			return file
-		safe s
-			| null s = False
-			| s == "." = False
-			| s == ".." = False
-			| otherwise = True
+		safe "" = False
+		safe "." = False
+		safe ".." = False
+		safe _ = True
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 495bf9fb63..5ae8354406 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -72,7 +72,7 @@ perform file oldkey newbackend = do
 				then do
 					-- Update symlink to use the new key.
 					liftIO $ removeFile file
-					next $ Command.Add.cleanup file newkey
+					next $ Command.Add.cleanup file newkey True
 				else stop
 	where
 		cleantmp t = whenM (doesFileExist t) $ removeFile t
diff --git a/debian/changelog b/debian/changelog
index 748a945d7b..fcc0b58b89 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 git-annex (3.20110720) UNRELEASED; urgency=low
 
   * Fix shell escaping in rsync special remote.
+  * addurl: --fast can be used to avoid immediately downloading the url.
 
  -- Joey Hess   Fri, 29 Jul 2011 15:27:30 +0200
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 11f617f1ba..2865c8af5d 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -282,6 +282,8 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
   Downloads each url to a file, which is added to the annex.
 
+  To avoid immediately downloading the url, specify --fast
+
 * fromkey file
 
   This plumbing-level command can be used to manually set up a file

From 614d8f98566ba8acc2bfeb661410b5d5eaec624c Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmFgsNxmnGznb5bbmcoWhoQOoxZZ-io61s"
 
Date: Mon, 8 Aug 2011 09:04:21 +0000
Subject: [PATCH 018/111] Added a comment

---
 ..._14be660aa57fadec0d81b32a8b52c66f._comment | 75 +++++++++++++++++++
 1 file changed, 75 insertions(+)
 create mode 100644 doc/bugs/Cabal_dependency_monadIO_missing/comment_1_14be660aa57fadec0d81b32a8b52c66f._comment

diff --git a/doc/bugs/Cabal_dependency_monadIO_missing/comment_1_14be660aa57fadec0d81b32a8b52c66f._comment b/doc/bugs/Cabal_dependency_monadIO_missing/comment_1_14be660aa57fadec0d81b32a8b52c66f._comment
new file mode 100644
index 0000000000..8e38205f00
--- /dev/null
+++ b/doc/bugs/Cabal_dependency_monadIO_missing/comment_1_14be660aa57fadec0d81b32a8b52c66f._comment
@@ -0,0 +1,75 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmFgsNxmnGznb5bbmcoWhoQOoxZZ-io61s"
+ nickname="Thomas"
+ subject="comment 1"
+ date="2011-08-08T09:04:20Z"
+ content="""
+I use Debian Squeeze, I have the Debian package cabal-install 0.8.0-1 installed.
+
+    $ git clone git://git-annex.branchable.com/
+    $ cd git-annex.branchable.com
+    $ cabal update
+    $ cabal install cabal-install
+
+This installed: Cabal-1.10.2.0, zlib-0.5.3.1, cabal-install 0.10.2.
+No version of monad-control or monadIO installed.
+
+    $ ~/.cabal/bin/cabal install
+    Registering QuickCheck-2.4.1.1...
+    Registering Crypto-4.2.3...
+    Registering base-unicode-symbols-0.2.2.1...
+    Registering deepseq-1.1.0.2...
+    Registering hxt-charproperties-9.1.0...
+    Registering hxt-regex-xmlschema-9.0.0...
+    Registering hxt-unicode-9.0.1...
+    Registering hxt-9.1.2...
+    Registering stm-2.2.0.1...
+    Registering hS3-0.5.6...
+    Registering transformers-0.2.2.0...
+    Registering monad-control-0.2.0.1...
+    [1 of 1] Compiling Main             ( Setup.hs, dist/setup/Main.o )
+    Linking ./dist/setup/setup ...
+    ghc -O2 -Wall -ignore-package monads-fd -fspec-constr-count=5 --make configure
+    [1 of 2] Compiling TestConfig       ( TestConfig.hs, TestConfig.o )
+    [2 of 2] Compiling Main             ( configure.hs, configure.o )
+    Linking configure ...
+    ./configure
+      checking version... 3.20110720
+      checking cp -a... yes
+      checking cp -p... yes
+      checking cp --reflink=auto... yes
+      checking uuid generator... uuid
+      checking xargs -0... yes
+      checking rsync... yes
+      checking curl... yes
+      checking bup... yes
+      checking gpg... yes
+      checking sha1... sha1sum
+      checking sha256... sha256sum
+      checking sha512... sha512sum
+      checking sha224... sha224sum
+      checking sha384... sha384sum
+
+    ...
+
+    Command/Add.hs:54:3:
+        No instance for (Control.Monad.IO.Control.MonadControlIO
+                           (Control.Monad.State.Lazy.StateT Annex.AnnexState IO))
+          arising from a use of `handle' at Command/Add.hs:54:3-24
+        Possible fix:
+          add an instance declaration for
+          (Control.Monad.IO.Control.MonadControlIO
+             (Control.Monad.State.Lazy.StateT Annex.AnnexState IO))
+        In the first argument of `($)', namely `handle (undo file key)'
+        In a stmt of a 'do' expression:
+              handle (undo file key) $ moveAnnex key file
+        In the expression:
+            do { handle (undo file key) $ moveAnnex key file;
+                 next $ cleanup file key }
+    cabal: Error: some packages failed to install:
+    git-annex-3.20110719 failed during the building phase. The exception was:
+    ExitFailure 1
+
+After I added a depencency for monadIO to the git-annex.cabal file, it installed correctly.  
+-- Thomas
+"""]]

From 065e1a507b74e8e41017a1a5368fc2b8aaac7f19 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawm4or5sJLWB0evPKp70Q2OND-JmFPnOkLA"
 
Date: Tue, 9 Aug 2011 13:00:21 +0000
Subject: [PATCH 019/111]

---
 .../--git-dir_and_--work-tree_options.mdwn    | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 doc/bugs/--git-dir_and_--work-tree_options.mdwn

diff --git a/doc/bugs/--git-dir_and_--work-tree_options.mdwn b/doc/bugs/--git-dir_and_--work-tree_options.mdwn
new file mode 100644
index 0000000000..2e5d1cf0c8
--- /dev/null
+++ b/doc/bugs/--git-dir_and_--work-tree_options.mdwn
@@ -0,0 +1,28 @@
+git-annex does not take into account the --git-dir and --work-tree command line options (while they can be useful when scripting).
+
+> mkdir /tmp/test
+> cd /tmp/test
+> git init
+Initialized empty Git repository in /tmp/test/.git/
+> git annex init test
+init test ok
+> touch foo
+> cd
+> git --git-dir=/tmp/test/.git --work-tree=/tmp/test annex add foo
+git-annex: Not in a git repository.
+
+regular git add works:
+> git --git-dir=/tmp/test/.git --work-tree=/tmp/test add foo
+> git --git-dir=/tmp/test/.git --work-tree=/tmp/test status 
+# On branch master
+#
+# Initial commit
+#
+# Changes to be committed:
+#   (use "git rm --cached ..." to unstage)
+#
+#       new file:   foo
+#
+
+git-annex version: 3.20110702
+

From 1e934c29d66f44ff074a9d02004e2bd6abb90462 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawm4or5sJLWB0evPKp70Q2OND-JmFPnOkLA"
 
Date: Tue, 9 Aug 2011 13:02:32 +0000
Subject: [PATCH 020/111]

---
 .../--git-dir_and_--work-tree_options.mdwn    | 43 ++++++++++---------
 1 file changed, 22 insertions(+), 21 deletions(-)

diff --git a/doc/bugs/--git-dir_and_--work-tree_options.mdwn b/doc/bugs/--git-dir_and_--work-tree_options.mdwn
index 2e5d1cf0c8..ebae56b049 100644
--- a/doc/bugs/--git-dir_and_--work-tree_options.mdwn
+++ b/doc/bugs/--git-dir_and_--work-tree_options.mdwn
@@ -1,28 +1,29 @@
 git-annex does not take into account the --git-dir and --work-tree command line options (while they can be useful when scripting).
 
-> mkdir /tmp/test
-> cd /tmp/test
-> git init
-Initialized empty Git repository in /tmp/test/.git/
-> git annex init test
-init test ok
-> touch foo
-> cd
-> git --git-dir=/tmp/test/.git --work-tree=/tmp/test annex add foo
-git-annex: Not in a git repository.
+    > mkdir /tmp/test
+    > cd /tmp/test
+    > git init
+    Initialized empty Git repository in /tmp/test/.git/
+    > git annex init test
+    init test ok
+    > touch foo
+    > cd
+    > git --git-dir=/tmp/test/.git --work-tree=/tmp/test annex add foo
+    git-annex: Not in a git repository.
 
 regular git add works:
-> git --git-dir=/tmp/test/.git --work-tree=/tmp/test add foo
-> git --git-dir=/tmp/test/.git --work-tree=/tmp/test status 
-# On branch master
-#
-# Initial commit
-#
-# Changes to be committed:
-#   (use "git rm --cached ..." to unstage)
-#
-#       new file:   foo
-#
+
+    > git --git-dir =/tmp/test/.git --work-tree=/tmp/test add foo
+    > git --git-dir=/tmp/test/.git --work-tree=/tmp/test status 
+    # On branch master
+    #
+    # Initial commit
+    #
+    # Changes to be committed:
+    #   (use "git rm --cached ..." to unstage)
+    #
+    #       new file:   foo
+    #
 
 git-annex version: 3.20110702
 

From 20a8f8c85bbda5d8af90782a596abbb1b0ac1e86 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawm4or5sJLWB0evPKp70Q2OND-JmFPnOkLA"
 
Date: Tue, 9 Aug 2011 13:04:34 +0000
Subject: [PATCH 021/111]

---
 doc/bugs/--git-dir_and_--work-tree_options.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/bugs/--git-dir_and_--work-tree_options.mdwn b/doc/bugs/--git-dir_and_--work-tree_options.mdwn
index ebae56b049..d76a42bfff 100644
--- a/doc/bugs/--git-dir_and_--work-tree_options.mdwn
+++ b/doc/bugs/--git-dir_and_--work-tree_options.mdwn
@@ -13,7 +13,7 @@ git-annex does not take into account the --git-dir and --work-tree command line
 
 regular git add works:
 
-    > git --git-dir =/tmp/test/.git --work-tree=/tmp/test add foo
+    > git --git-dir=/tmp/test/.git --work-tree=/tmp/test add foo
     > git --git-dir=/tmp/test/.git --work-tree=/tmp/test status 
     # On branch master
     #

From f8282b55cece7561cd04788637627e79d2f29e17 Mon Sep 17 00:00:00 2001
From: "http://christian.amsuess.com/chrysn" 
Date: Wed, 10 Aug 2011 16:56:51 +0000
Subject: [PATCH 022/111] problems with version 3 upgrade instructions

---
 doc/forum/version_3_upgrade.mdwn | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/forum/version_3_upgrade.mdwn

diff --git a/doc/forum/version_3_upgrade.mdwn b/doc/forum/version_3_upgrade.mdwn
new file mode 100644
index 0000000000..476b8c235f
--- /dev/null
+++ b/doc/forum/version_3_upgrade.mdwn
@@ -0,0 +1,9 @@
+after upgrading to git-annex 3, i'm stuck with diverging git-annex branches -- i didn't manage to follow this line in the directions:
+
+> After this upgrade, you should make sure you include the git-annex branch when git pushing and pulling.
+
+could you explain how to do that in a littel more detail? git pull seems to only merge master, although i have these ``.git/config`` settings:
+
+    [branch "git-annex"]
+    	remote = poseidon
+    	merge = git-annex

From 8b847517a810d384a79178124b9766141b89bc17 Mon Sep 17 00:00:00 2001
From: "http://christian.amsuess.com/chrysn" 
Date: Wed, 10 Aug 2011 17:00:09 +0000
Subject: [PATCH 023/111] (i don't have a remote origin, but it's more common
 and doesn't change the example)

---
 doc/forum/version_3_upgrade.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/forum/version_3_upgrade.mdwn b/doc/forum/version_3_upgrade.mdwn
index 476b8c235f..7fdbcbc805 100644
--- a/doc/forum/version_3_upgrade.mdwn
+++ b/doc/forum/version_3_upgrade.mdwn
@@ -5,5 +5,5 @@ after upgrading to git-annex 3, i'm stuck with diverging git-annex branches -- i
 could you explain how to do that in a littel more detail? git pull seems to only merge master, although i have these ``.git/config`` settings:
 
     [branch "git-annex"]
-    	remote = poseidon
+    	remote = origin
     	merge = git-annex

From 18012c7ec8e8bcbcdc8712be29046c12adeaf6d2 Mon Sep 17 00:00:00 2001
From: "http://christian.amsuess.com/chrysn" 
Date: Fri, 12 Aug 2011 13:09:11 +0000
Subject: [PATCH 024/111] enhancement suggestion

---
 ...e_commit_messages_in_git-annex_branch.mdwn | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn

diff --git a/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn b/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn
new file mode 100644
index 0000000000..68b24c2fba
--- /dev/null
+++ b/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn
@@ -0,0 +1,26 @@
+as of git-annex version 3.20110719, all git-annex commits only contain the word "update" as a commit message. given that the contents of the commit are pretty non-descriptive (SHA1 hashes for file names, uuids for repository names), i suggest to have more descriptive commit messages, as shown here:
+
+    /mnt/usb_disk/photos/2011$ git annex get
+    /mnt/usb_disk/photos/2011$ git show git-annex
+    [...]
+    usb-disk-photos: get 2011
+    
+    * 10 files retrieved from 2 sources (9 from local-harddisk, 1 from my-server)
+    * 120 files were already present
+    * 2 files could not be retrieved
+    /mnt/usb_disk/photos/2011$ cd ~/photos/2011/07
+    ~/photos/2011/07$ git copy --to my-server
+    ~/photos/2011/07$ git show git-annex
+    [...]
+    local-harddisk: copy 2011/07 to my-server
+    
+    * 20 files pushed
+    ~/photos/2011/07$
+
+in my opinion, the messages should at least contain
+
+* what command was used
+* in which repository they were executed
+* which files or directories they affected (not necessarily all files, but what was given on command line or implicitly from the working directory)
+
+--[[chrysn]]

From 829ea31b72b1a9c42d770815363ef350ec2bda31 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 15 Aug 2011 13:30:59 -0400
Subject: [PATCH 025/111] typo

---
 doc/internals.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index a4ec5c417b..e80ecbac0d 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -22,7 +22,7 @@ deleting or changing the file contents.
 This branch is managed by git-annex, with the contents listed below.
 
 The file `.git/annex/index` is a separate git index file it uses
-to accumlate changes for the git-annex. Also, `.git/annex/journal/` is used
+to accumulate changes for the git-annex. Also, `.git/annex/journal/` is used
 to record changes before they are added to git.
 
 Note that for speed reasons, git-annex assumes only it will modify this

From 9f719e5674280f49b71c4d58dcc0b0002845f852 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkHscTHMCNvjJ6nLI1VpsBrJFI5FTwhUT4"
 
Date: Mon, 15 Aug 2011 18:18:22 +0000
Subject: [PATCH 026/111]

---
 ...ith_the_annex_directory_exposed_to_http.mdwn | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn

diff --git a/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn b/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn
new file mode 100644
index 0000000000..c1b790be1e
--- /dev/null
+++ b/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn
@@ -0,0 +1,17 @@
+Let's say that http://people.collabora.com/~alsuren/git/fate-suite.git/ is a bare git repo. It has been 'git update-server-info'd so that it can be served on a dumb http server.
+
+The repo is also a git annex remote, created using the following commands:
+
+* git remote add alsuren git+ssh://people.collabora.co.uk/user/alsuren/public_html/fate-suite.git
+* git push alsuren --all
+* git annex copy --to=alsuren
+
+so http://people.collabora.com/~alsuren/git/fate-suite.git/annex is a valid git annex (though
+
+I would like to be able to use the following commands to get a clone of the repo:
+
+* git clone http://people.collabora.com/~alsuren/git/fate-suite.git/
+* cd fate-suite
+* git annex get
+
+This would allow contributors to quickly get a copy of our upstream repo and start contributing with minimal bandwidth/effort.

From c3f74c6959cf67ad511dc9b6bed93112c754c247 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkHscTHMCNvjJ6nLI1VpsBrJFI5FTwhUT4"
 
Date: Mon, 15 Aug 2011 18:19:47 +0000
Subject: [PATCH 027/111]

---
 ...git_repo__44___with_the_annex_directory_exposed_to_http.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn b/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn
index c1b790be1e..a2e344c115 100644
--- a/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn
+++ b/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn
@@ -6,7 +6,7 @@ The repo is also a git annex remote, created using the following commands:
 * git push alsuren --all
 * git annex copy --to=alsuren
 
-so http://people.collabora.com/~alsuren/git/fate-suite.git/annex is a valid git annex (though
+so http://people.collabora.com/~alsuren/git/fate-suite.git/annex is a valid git annex (though listing dirs is forbidden, so you need to know the filenames ahead of time).
 
 I would like to be able to use the following commands to get a clone of the repo:
 

From c04f1f4c30d5334b06b606c89fcb8e693bd88701 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 16:11:55 -0400
Subject: [PATCH 028/111] response

---
 ...escriptive_commit_messages_in_git-annex_branch.mdwn | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn b/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn
index 68b24c2fba..aad119ffd3 100644
--- a/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn
+++ b/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn
@@ -24,3 +24,13 @@ in my opinion, the messages should at least contain
 * which files or directories they affected (not necessarily all files, but what was given on command line or implicitly from the working directory)
 
 --[[chrysn]]
+
+> The implementation of the git-annex branch precludes more descriptive
+> commit messages, since a single commit can include changes that were
+> previously staged to the branch's index file, or spooled to its journal
+> by other git-annex commands (either concurrently running or
+> interrupted commands, or even changes needed to automatically merge
+> other git-annex branches).
+> 
+> It would be possible to make it *less* verbose, with an empty commit
+> message. :) --[[Joey]] 

From cfcd7805b441c48e404826903480113a82cff9cf Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 19:23:56 -0400
Subject: [PATCH 029/111] add repoIsHttp

---
 Git.hs | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/Git.hs b/Git.hs
index 9b7ac7ea91..b226bce0f2 100644
--- a/Git.hs
+++ b/Git.hs
@@ -17,6 +17,7 @@ module Git (
 	localToUrl,
 	repoIsUrl,
 	repoIsSsh,
+	repoIsHttp,
 	repoIsLocalBare,
 	repoDescribe,
 	repoLocation,
@@ -206,6 +207,13 @@ repoIsSsh Repo { location = Url url }
 	| otherwise = False
 repoIsSsh _ = False
 
+repoIsHttp :: Repo -> Bool
+repoIsHttp Repo { location = Url url } 
+	| uriScheme url == "http:" = True
+	| uriScheme url == "https:" = True
+	| otherwise = False
+repoIsHttp _ = False
+
 configAvail ::Repo -> Bool
 configAvail Repo { config = c } = c /= M.empty
 

From 354c5f349bcd8c43b45191983dce4a6c7489e9ed Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 20:45:58 -0400
Subject: [PATCH 030/111] add withTempFile

This is essentially the same as withSystemTempFile from System.IO.Temp,
but that library is not packaged for Debian, and may not be widely used.
I see various other withTempFile implementations here and there, none canonical.
Sigh.
---
 Utility.hs | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/Utility.hs b/Utility.hs
index dab3b4d0ea..511350898a 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -23,6 +23,7 @@ module Utility (
 	unsetFileMode,
 	readMaybe,
 	viaTmp,
+	withTempFile,
 	dirContains,
 	dirContents,
 	myHomeDir,
@@ -38,6 +39,7 @@ module Utility (
 	prop_relPathDirToFile_basics
 ) where
 
+import IO (bracket)
 import System.IO
 import System.Exit
 import qualified System.Posix.Process
@@ -253,6 +255,18 @@ viaTmp a file content = do
 	a tmpfile content
 	renameFile tmpfile file
 
+{- Runs an action with a temp file, then removes the file. -}
+withTempFile :: String -> (FilePath -> Handle -> IO a) -> IO a
+withTempFile template action = bracket create remove use
+	where
+		create = do
+			tmpdir <- catch getTemporaryDirectory (const $ return ".")
+			openTempFile tmpdir template
+		remove (name, handle) = do
+			hClose handle
+			catchBool (removeFile name >> return True)
+		use (name, handle) = action name handle
+
 {- Lists the contents of a directory.
  - Unlike getDirectoryContents, paths are not relative to the directory. -}
 dirContents :: FilePath -> IO [FilePath]

From 5000aba76e6f066fd310d9635ea4369f07684b86 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 20:47:48 -0400
Subject: [PATCH 031/111] avoid namespace conflict

---
 Command.hs        | 2 --
 Command/SetKey.hs | 2 +-
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/Command.hs b/Command.hs
index 02bbd29d44..d3c1640ee0 100644
--- a/Command.hs
+++ b/Command.hs
@@ -178,8 +178,6 @@ withKeys :: CommandSeekKeys
 withKeys a params = return $ map (a . parse) params
 	where
 		parse p = fromMaybe (error "bad key") $ readKey p
-withTempFile :: CommandSeekStrings
-withTempFile a params = return $ map a params
 withNothing :: CommandSeekNothing
 withNothing a [] = return [a]
 withNothing _ _ = error "This command takes no parameters."
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index f2a5259bac..807cbd5b93 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -20,7 +20,7 @@ command = [repoCommand "setkey" paramPath seek
 	"sets annexed content for a key using a temp file"]
 
 seek :: [CommandSeek]
-seek = [withTempFile start]
+seek = [withStrings start]
 
 {- Sets cached content for a key. -}
 start :: CommandStartString

From 07f2e7ee726f3d7f60cd478e928afc69db60c0c8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 20:48:11 -0400
Subject: [PATCH 032/111] support reading git config from http remotes

The config file is downloaded to a temp file, and git-config run on that
to parse it.
---
 Remote/Git.hs | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index de51c891e2..c8facb47a7 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -12,6 +12,7 @@ import Control.Monad.State (liftIO)
 import qualified Data.Map as M
 import System.Cmd.Utils
 import System.Posix.Files
+import System.IO
 
 import Types
 import Types.Remote
@@ -24,7 +25,8 @@ import qualified Content
 import Messages
 import Utility.CopyFile
 import Utility.RsyncFile
-import Remote.Ssh
+import Remote.Helper.Ssh
+import qualified Remote.Helper.Url as Url
 import Config
 
 remote :: RemoteType Annex
@@ -75,6 +77,7 @@ tryGitConfigRead :: Git.Repo -> Annex Git.Repo
 tryGitConfigRead r 
 	| not $ M.null $ Git.configMap r = return r -- already read
 	| Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" []
+	| Git.repoIsHttp r = store $ safely $ geturlconfig
 	| Git.repoIsUrl r = return r
 	| otherwise = store $ safely $ Git.configRead r
 	where
@@ -85,9 +88,19 @@ tryGitConfigRead r
 			case result of
 				Left _ -> return r
 				Right r' -> return r'
+
 		pipedconfig cmd params = safely $
 			pOpen ReadFromPipe cmd (toCommand params) $
 				Git.hConfigRead r
+
+		geturlconfig = do
+			s <- Url.get (Git.repoLocation r ++ "/config")
+			withTempFile "git-annex.tmp" $ \tmpfile -> \h -> do
+				hPutStr h s
+				hClose h
+				pOpen ReadFromPipe "git" ["config", "--list", "--file", tmpfile] $
+					Git.hConfigRead r
+
 		store a = do
 			r' <- a
 			g <- Annex.gitRepo
@@ -95,6 +108,7 @@ tryGitConfigRead r
 			let g' = Git.remotesAdd g $ exchange l r'
 			Annex.changeState $ \s -> s { Annex.repo = g' }
 			return r'
+
 		exchange [] _ = []
 		exchange (old:ls) new =
 			if Git.repoRemoteName old == Git.repoRemoteName new

From 4545a0e78cf675c6bbbcdd86b5c06bf99bb0c7e9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 20:49:04 -0400
Subject: [PATCH 033/111] split out generic url stuff into a helper library
 from Remote.Web

---
 Command/AddUrl.hs    |  3 +-
 Remote/Helper/Url.hs | 66 ++++++++++++++++++++++++++++++++++++++++++++
 Remote/Web.hs        | 42 ++++++----------------------
 3 files changed, 77 insertions(+), 34 deletions(-)
 create mode 100644 Remote/Helper/Url.hs

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index e87de384b3..add71c8209 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -14,6 +14,7 @@ import System.Directory
 
 import Command
 import qualified Backend
+import qualified Remote.Helper.Url
 import qualified Remote.Web
 import qualified Command.Add
 import qualified Annex
@@ -52,7 +53,7 @@ download url file = do
 	let dummykey = Backend.URL.fromUrl url
 	let tmp = gitAnnexTmpLocation g dummykey
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
-	ok <- Remote.Web.download [url] tmp
+	ok <- Remote.Helper.Url.download url tmp
 	if ok
 		then do
 			[(_, backend)] <- Backend.chooseBackends [file]
diff --git a/Remote/Helper/Url.hs b/Remote/Helper/Url.hs
new file mode 100644
index 0000000000..d3aea56220
--- /dev/null
+++ b/Remote/Helper/Url.hs
@@ -0,0 +1,66 @@
+{- Url downloading for remotes.
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Remote.Helper.Url (
+	exists,
+	download,
+	get
+) where
+
+import Control.Monad (liftM)
+import Control.Monad.State (liftIO)
+import qualified Network.Browser as Browser
+import Network.HTTP
+import Network.URI
+
+import Types
+import Messages
+import Utility
+
+type URLString = String
+
+{- Checks that an url exists and could be successfully downloaded. -}
+exists :: URLString -> IO Bool
+exists url =
+	case parseURI url of
+		Nothing -> return False
+		Just u -> do
+			r <- request u HEAD
+			case rspCode r of
+				(2,_,_) -> return True
+				_ -> return False
+
+{- Used to download large files, such as the contents of keys.
+ - Uses curl program for its progress bar. -}
+download :: URLString -> FilePath -> Annex Bool
+download url file = do
+	showOutput -- make way for curl progress bar
+	liftIO $ boolSystem "curl" [Params "-L -C - -# -o", File file, File url]
+
+{- Downloads a small file. -}
+get :: URLString -> IO String
+get url =
+	case parseURI url of
+		Nothing -> error "url parse error"
+		Just u -> do
+			r <- request u GET
+			case rspCode r of
+				(2,_,_) -> return $ rspBody r
+				_ -> error $ rspReason r
+
+{- Makes a http request of an url. For example, HEAD can be used to
+ - check if the url exists, or GET used to get the url content (best for
+ - small urls). -}
+request :: URI -> RequestMethod -> IO (Response String)
+request url requesttype = Browser.browse $ do
+	Browser.setErrHandler ignore
+	Browser.setOutHandler ignore
+	Browser.setAllowRedirects True
+	liftM snd $ Browser.request
+		(mkRequest requesttype url :: Request_String)
+	where
+		ignore = const $ return ()
diff --git a/Remote/Web.hs b/Remote/Web.hs
index cd028a06d9..cc96d5306d 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -7,28 +7,24 @@
 
 module Remote.Web (
 	remote,
-	setUrl,
-	download
+	setUrl
 ) where
 
 import Control.Monad.State (liftIO)
 import Control.Exception
 import System.FilePath
-import Network.Browser
-import Network.HTTP
-import Network.URI
 
 import Types
 import Types.Remote
 import qualified Git
 import qualified Annex
 import Messages
-import Utility
 import UUID
 import Config
 import PresenceLog
 import LocationLog
 import Locations
+import qualified Remote.Helper.Url as Url
 
 type URLString = String
 
@@ -90,9 +86,12 @@ setUrl key url status = do
 	logChange g key webUUID (if null us then InfoMissing else InfoPresent)
 
 downloadKey :: Key -> FilePath -> Annex Bool
-downloadKey key file = do
-	us <- getUrls key
-	download us file
+downloadKey key file = iter =<< getUrls key
+	where
+		iter [] = return False
+		iter (url:urls) = do
+			ok <- Url.download url file
+			if ok then return ok else iter urls
 
 uploadKey :: Key -> Annex Bool
 uploadKey _ = do
@@ -114,28 +113,5 @@ checkKey' :: [URLString] -> Annex Bool
 checkKey' [] = return False
 checkKey' (u:us) = do
 	showAction $ "checking " ++ u
-	e <- liftIO $ urlexists u
+	e <- liftIO $ Url.exists u
 	if e then return e else checkKey' us
-
-urlexists :: URLString -> IO Bool
-urlexists url =
-	case parseURI url of
-		Nothing -> return False
-		Just u -> do
-			(_, r) <- Network.Browser.browse $ do
-				setErrHandler ignore
-				setOutHandler ignore
-				setAllowRedirects True
-				request (mkRequest HEAD u :: Request_String)
-			case rspCode r of
-				(2,_,_) -> return True
-				_ -> return False
-	where
-		ignore = const $ return ()
-
-download :: [URLString] -> FilePath -> Annex Bool
-download [] _ = return False
-download (url:us) file = do
-	showOutput -- make way for curl progress bar
-	ok <- liftIO $ boolSystem "curl" [Params "-L -C - -# -o", File file, File url]
-	if ok then return ok else download us file

From a55faff08fd9173edaf22a1de46cf7fafe89ebb7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 20:49:54 -0400
Subject: [PATCH 034/111] reorg Remote/*

---
 Command/Map.hs                     | 2 +-
 Remote/Bup.hs                      | 6 +++---
 Remote/Directory.hs                | 4 ++--
 Remote/{ => Helper}/Encryptable.hs | 2 +-
 Remote/{ => Helper}/Special.hs     | 2 +-
 Remote/{ => Helper}/Ssh.hs         | 2 +-
 Remote/Hook.hs                     | 4 ++--
 Remote/Rsync.hs                    | 4 ++--
 Remote/S3real.hs                   | 4 ++--
 9 files changed, 15 insertions(+), 15 deletions(-)
 rename Remote/{ => Helper}/Encryptable.hs (98%)
 rename Remote/{ => Helper}/Special.hs (97%)
 rename Remote/{ => Helper}/Ssh.hs (98%)

diff --git a/Command/Map.hs b/Command/Map.hs
index 07f127f14e..75c5b0b550 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -22,7 +22,7 @@ import Types
 import Utility
 import UUID
 import Trust
-import Remote.Ssh
+import Remote.Helper.Ssh
 import qualified Utility.Dot as Dot
 
 -- a link from the first repository to the second (its remote)
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 1023cda186..c82f84745d 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -30,9 +30,9 @@ import Locations
 import Config
 import Utility
 import Messages
-import Remote.Ssh
-import Remote.Special
-import Remote.Encryptable
+import Remote.Helper.Ssh
+import Remote.Helper.Special
+import Remote.Helper.Encryptable
 import Crypto
 
 type BupRepo = String
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 235f613000..fd227f85d8 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -27,8 +27,8 @@ import Utility.CopyFile
 import Config
 import Content
 import Utility
-import Remote.Special
-import Remote.Encryptable
+import Remote.Helper.Special
+import Remote.Helper.Encryptable
 import Crypto
 
 remote :: RemoteType Annex
diff --git a/Remote/Encryptable.hs b/Remote/Helper/Encryptable.hs
similarity index 98%
rename from Remote/Encryptable.hs
rename to Remote/Helper/Encryptable.hs
index 66e1738ac2..04041c6553 100644
--- a/Remote/Encryptable.hs
+++ b/Remote/Helper/Encryptable.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Remote.Encryptable where
+module Remote.Helper.Encryptable where
 
 import qualified Data.Map as M
 import Control.Monad.State (liftIO)
diff --git a/Remote/Special.hs b/Remote/Helper/Special.hs
similarity index 97%
rename from Remote/Special.hs
rename to Remote/Helper/Special.hs
index d6f362ce30..c302a0ff5e 100644
--- a/Remote/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Remote.Special where
+module Remote.Helper.Special where
 
 import qualified Data.Map as M
 import Data.Maybe
diff --git a/Remote/Ssh.hs b/Remote/Helper/Ssh.hs
similarity index 98%
rename from Remote/Ssh.hs
rename to Remote/Helper/Ssh.hs
index fe4e6dfc1a..478b018812 100644
--- a/Remote/Ssh.hs
+++ b/Remote/Helper/Ssh.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Remote.Ssh where
+module Remote.Helper.Ssh where
 
 import Control.Monad.State (liftIO)
 
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 87f86ffe4f..ef52d0482b 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -28,8 +28,8 @@ import Locations
 import Config
 import Content
 import Utility
-import Remote.Special
-import Remote.Encryptable
+import Remote.Helper.Special
+import Remote.Helper.Encryptable
 import Crypto
 import Messages
 
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 9535376097..eba67e3fd6 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -26,8 +26,8 @@ import Locations
 import Config
 import Content
 import Utility
-import Remote.Special
-import Remote.Encryptable
+import Remote.Helper.Special
+import Remote.Helper.Encryptable
 import Crypto
 import Messages
 import Utility.RsyncFile
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index e4dcc2a71d..456a77f0e4 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -33,8 +33,8 @@ import UUID
 import Messages
 import Locations
 import Config
-import Remote.Special
-import Remote.Encryptable
+import Remote.Helper.Special
+import Remote.Helper.Encryptable
 import Crypto
 import Content
 import Utility.Base64

From 5ccb926b51b0a270c8b1d754dac78d2074e07bdf Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 21:04:23 -0400
Subject: [PATCH 035/111] support for getting files from http git remotes

---
 Remote/Git.hs | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index c8facb47a7..1adf8cfeb7 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -119,9 +119,10 @@ tryGitConfigRead r
  - If the remote cannot be accessed, returns a Left error.
  -}
 inAnnex :: Git.Repo -> Key -> Annex (Either IOException Bool)
-inAnnex r key = if Git.repoIsUrl r
-		then checkremote
-		else liftIO (try checklocal ::IO (Either IOException Bool))
+inAnnex r key
+	| Git.repoIsHttp r = safely checkhttp
+	| Git.repoIsUrl r = checkremote
+	| otherwise = safely checklocal
 	where
 		checklocal = do
 			-- run a local check inexpensively,
@@ -133,7 +134,12 @@ inAnnex r key = if Git.repoIsUrl r
 			inannex <- onRemote r (boolSystem, False) "inannex" 
 				[Param (show key)]
 			return $ Right inannex
-	
+		checkhttp = Url.exists $ keyUrl r key
+		safely a = liftIO (try a ::IO (Either IOException Bool))
+
+keyUrl :: Git.Repo -> Key -> String
+keyUrl r key = Git.repoLocation r ++ "/" ++ annexLocation key
+
 dropKey :: Git.Repo -> Key -> Annex Bool
 dropKey r key = 
 	onRemote r (boolSystem, False) "dropkey"
@@ -146,8 +152,9 @@ copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool
 copyFromRemote r key file
 	| not $ Git.repoIsUrl r = rsyncOrCopyFile r (gitAnnexLocation r key) file
 	| Git.repoIsSsh r = rsyncHelper =<< rsyncParamsRemote r True key file
-	| otherwise = error "copying from non-ssh repo not supported"
-		
+	| Git.repoIsHttp r = Url.download (keyUrl r key) file
+	| otherwise = error "copying from non-ssh, non-http repo not supported"
+
 {- Tries to copy a key's content to a remote's annex. -}
 copyToRemote :: Git.Repo -> Key -> Annex Bool
 copyToRemote r key

From e6752cc06424df18f3e14406674f9a630c3387ef Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 21:12:48 -0400
Subject: [PATCH 036/111] Added support for getting content from git remotes
 using http (and https).

---
 debian/changelog                                             | 1 +
 ..._repo__44___with_the_annex_directory_exposed_to_http.mdwn | 3 +++
 doc/special_remotes.mdwn                                     | 5 +++--
 doc/special_remotes/web.mdwn                                 | 4 ++++
 4 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index fcc0b58b89..89cc794e7e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (3.20110720) UNRELEASED; urgency=low
 
   * Fix shell escaping in rsync special remote.
   * addurl: --fast can be used to avoid immediately downloading the url.
+  * Added support for getting content from git remotes using http (and https).
 
  -- Joey Hess   Fri, 29 Jul 2011 15:27:30 +0200
 
diff --git a/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn b/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn
index a2e344c115..ba7dcad300 100644
--- a/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn
+++ b/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn
@@ -15,3 +15,6 @@ I would like to be able to use the following commands to get a clone of the repo
 * git annex get
 
 This would allow contributors to quickly get a copy of our upstream repo and start contributing with minimal bandwidth/effort.
+
+> This is now supported.. I look forward to seeing your project using it!
+> --[[Joey]] [[!tag done]]
diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index afc6a2cf23..55cd1f1a0f 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -1,6 +1,7 @@
 Most [[backends]] can transfer data to and from configured git remotes.
-Normally those remotes are normal git repositories (bare and non-bare),
-that store the file contents in their own git annex directory.
+Normally those remotes are normal git repositories (bare and non-bare;
+local and remote), that store the file contents in their own git annex
+directory.
 
 But, git-annex also extends git's concept of remotes, with these special
 types of remotes. These can be used just like any normal remote by git-annex.
diff --git a/doc/special_remotes/web.mdwn b/doc/special_remotes/web.mdwn
index 6991ce4e44..a969fb071d 100644
--- a/doc/special_remotes/web.mdwn
+++ b/doc/special_remotes/web.mdwn
@@ -5,3 +5,7 @@ See [[walkthrough/using_the_web]] for usage examples.
 
 Currently git-annex only supports downloading content from the web; 
 it cannot upload to it or remove content.
+
+This special remote uses arbitrary urls on the web as the source for content.
+git-annex can also download content from a normal git remote, accessible by
+http.

From 790b0f38795ce9b62deda416c458376f1e7b7016 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 21:16:23 -0400
Subject: [PATCH 037/111] update

---
 doc/transferring_data.mdwn | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/doc/transferring_data.mdwn b/doc/transferring_data.mdwn
index f6ae9bfcde..57873f6f0e 100644
--- a/doc/transferring_data.mdwn
+++ b/doc/transferring_data.mdwn
@@ -1,8 +1,12 @@
 git-annex can transfer data to or from any of a repository's git remotes.
 Depending on where the remote is, the data transfer is done using rsync
-(over ssh, with automatic resume), or plain cp (with copy-on-write
-optimisations on supported filesystems). Some [[special_remotes]]
-are also supported that are not traditional git remotes.
+(over ssh or locally), or plain cp (with copy-on-write
+optimisations on supported filesystems), or using curl (for repositories
+on the web). Some [[special_remotes]] are also supported that are not
+traditional git remotes.
+
+If a data transfer is interrupted, git-annex retains the partial transfer
+to allow it to be automatically resumed later.
 
 It's equally easy to transfer a single file to or from a repository,
 or to launch a retrievel of a massive pile of files from whatever

From f5449aae16af431ce6474080d123a930209b2cde Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 21:20:14 -0400
Subject: [PATCH 038/111] error out when dropping from http repo

---
 Remote/Git.hs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index 1adf8cfeb7..d4847d6105 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -141,8 +141,9 @@ keyUrl :: Git.Repo -> Key -> String
 keyUrl r key = Git.repoLocation r ++ "/" ++ annexLocation key
 
 dropKey :: Git.Repo -> Key -> Annex Bool
-dropKey r key = 
-	onRemote r (boolSystem, False) "dropkey"
+dropKey r key
+	| Git.repoIsHttp r = error "dropping from http repo not supported"
+	| otherwise = onRemote r (boolSystem, False) "dropkey"
 		[ Params "--quiet --force"
 		, Param $ show key
 		]

From dffbf49d90236aedb559a3db26a96e2ae7904ece Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 17 Aug 2011 01:33:08 +0000
Subject: [PATCH 039/111] Added a comment

---
 ...ment_1_05fc9c9cad26c520bebb98c852c71e35._comment | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/forum/version_3_upgrade/comment_1_05fc9c9cad26c520bebb98c852c71e35._comment

diff --git a/doc/forum/version_3_upgrade/comment_1_05fc9c9cad26c520bebb98c852c71e35._comment b/doc/forum/version_3_upgrade/comment_1_05fc9c9cad26c520bebb98c852c71e35._comment
new file mode 100644
index 0000000000..18746225e9
--- /dev/null
+++ b/doc/forum/version_3_upgrade/comment_1_05fc9c9cad26c520bebb98c852c71e35._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-08-17T01:33:08Z"
+ content="""
+It's ok that `git pull` does not merge the git-annex branch. You can merge it with `git annex merge`, or it will be done
+automatically when you use other git-annex commands.
+
+If you use `git pull` and `git push` without any options, the defaults will make git pull and push the git-annex branch automatically.
+
+But if you're in the habit of doing `git push origin master`, that won't cause the git-annex branch to be pushed (use `git push origin git-annex` to manually push it then). Similarly, `git pull origin master` won't pull it. And also, the `remote.origin.fetch` setting in `.git/config` can be modified in ways that make `git pull` not automatically pull the git-annex branch. So those are the things to avoid after upgrade to v3, basically.
+"""]]

From 4a023dd1aafb725ba5ccff5f09adc7055a2643ba Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 22:22:00 -0400
Subject: [PATCH 040/111] Added curl to Debian package dependencies.

---
 debian/changelog | 1 +
 debian/control   | 1 +
 2 files changed, 2 insertions(+)

diff --git a/debian/changelog b/debian/changelog
index 89cc794e7e..dc03647a72 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,7 @@ git-annex (3.20110720) UNRELEASED; urgency=low
   * Fix shell escaping in rsync special remote.
   * addurl: --fast can be used to avoid immediately downloading the url.
   * Added support for getting content from git remotes using http (and https).
+  * Added curl to Debian package dependencies.
 
  -- Joey Hess   Fri, 29 Jul 2011 15:27:30 +0200
 
diff --git a/debian/control b/debian/control
index d519c42b71..975faf5ea5 100644
--- a/debian/control
+++ b/debian/control
@@ -31,6 +31,7 @@ Depends: ${misc:Depends}, ${shlibs:Depends},
 	git | git-core,
 	uuid,
 	rsync,
+	curl,
 	openssh-client
 Suggests: graphviz, bup, gnupg
 Description: manage files with git, without checking their contents into git

From cde8221051546902ef24a4fa51f2108177b67a4a Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 17 Aug 2011 04:56:30 +0000
Subject: [PATCH 041/111] Added a comment

---
 .../comment_2_4f4d8e1e00a2a4f7e8a8ab082e16adac._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Cabal_dependency_monadIO_missing/comment_2_4f4d8e1e00a2a4f7e8a8ab082e16adac._comment

diff --git a/doc/bugs/Cabal_dependency_monadIO_missing/comment_2_4f4d8e1e00a2a4f7e8a8ab082e16adac._comment b/doc/bugs/Cabal_dependency_monadIO_missing/comment_2_4f4d8e1e00a2a4f7e8a8ab082e16adac._comment
new file mode 100644
index 0000000000..adf7a34e66
--- /dev/null
+++ b/doc/bugs/Cabal_dependency_monadIO_missing/comment_2_4f4d8e1e00a2a4f7e8a8ab082e16adac._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-08-17T04:56:30Z"
+ content="""
+Finally got a chance to try to reproduce this. I followed your recipe exactly in a clean squeeze chroot. monadIO was not installed, but git-annex built ok, using monad-control.
+"""]]

From f0c21307001ec03437cca066d21f35f541b66de9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 01:34:15 -0400
Subject: [PATCH 042/111] releasing version 3.20110817

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index dc03647a72..3eab43578a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,11 +1,11 @@
-git-annex (3.20110720) UNRELEASED; urgency=low
+git-annex (3.20110817) unstable; urgency=low
 
   * Fix shell escaping in rsync special remote.
   * addurl: --fast can be used to avoid immediately downloading the url.
   * Added support for getting content from git remotes using http (and https).
   * Added curl to Debian package dependencies.
 
- -- Joey Hess   Fri, 29 Jul 2011 15:27:30 +0200
+ -- Joey Hess   Wed, 17 Aug 2011 01:29:02 -0400
 
 git-annex (3.20110719) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 807158f3eb..14188cd350 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110719
+Version: 3.20110817
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 00cc34a4c3a496f4d8fb5dfc5de826e321e6ac56 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 01:34:43 -0400
Subject: [PATCH 043/111] add news item for git-annex 3.20110817

---
 doc/news/version_3.20110624.mdwn | 33 --------------------------------
 doc/news/version_3.20110817.mdwn |  6 ++++++
 2 files changed, 6 insertions(+), 33 deletions(-)
 delete mode 100644 doc/news/version_3.20110624.mdwn
 create mode 100644 doc/news/version_3.20110817.mdwn

diff --git a/doc/news/version_3.20110624.mdwn b/doc/news/version_3.20110624.mdwn
deleted file mode 100644
index 6204673bd2..0000000000
--- a/doc/news/version_3.20110624.mdwn
+++ /dev/null
@@ -1,33 +0,0 @@
-News for git-annex 3.20110624:
-
-There has been another change to the git-annex data store.
-Use `git annex upgrade` to migrate your repositories to the new
-layout. See [[upgrades]].
-
-The significant change this time is that the .git-annex/ directory
-is gone; instead there is a git-annex branch that is automatically
-maintained by git-annex, and encapsulates all its state nicely out
-of your way.
-
-You should make sure you include the git-annex branch when
-git pushing and pulling.
-
-git-annex 3.20110624 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * New repository format, annex.version=3. Use `git annex upgrade` to migrate.
-   * git-annex now stores its logs in a git-annex branch.
-   * merge: New subcommand. Auto-merges the new git-annex branch.
-   * Improved handling of bare git repos with annexes. Many more commands will
-     work in them.
-   * git-annex is now more robust; it will never leave state files
-     uncommitted when some other git process comes along and locks the index
-     at an inconvenient time.
-   * rsync is now used when copying files from repos on other filesystems.
-     cp is still used when copying file from repos on the same filesystem,
-     since --reflink=auto can make it significantly faster on filesystems
-     such as btrfs.
-   * Allow --trust etc to specify a repository by name, for temporarily
-     trusting repositories that are not configured remotes.
-   * unlock: Made atomic.
-   * git-union-merge: New git subcommand, that does a generic union merge
-     operation, and operates efficiently without touching the working tree."""]]
diff --git a/doc/news/version_3.20110817.mdwn b/doc/news/version_3.20110817.mdwn
new file mode 100644
index 0000000000..51388f3c78
--- /dev/null
+++ b/doc/news/version_3.20110817.mdwn
@@ -0,0 +1,6 @@
+git-annex 3.20110817 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Fix shell escaping in rsync special remote.
+   * addurl: --fast can be used to avoid immediately downloading the url.
+   * Added support for getting content from git remotes using http (and https).
+   * Added curl to Debian package dependencies."""]]
\ No newline at end of file

From e950947cc9af07e03d4d2c335154790bc2943016 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawla7u6eLKNYZ09Z7xwBffqLaXquMQC07fU"
 
Date: Wed, 17 Aug 2011 12:34:48 +0000
Subject: [PATCH 044/111] Added a comment: squeeze-backports update?

---
 .../comment_1_029486088d098c2d4f1099f2f0e701a9._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/install/Debian/comment_1_029486088d098c2d4f1099f2f0e701a9._comment

diff --git a/doc/install/Debian/comment_1_029486088d098c2d4f1099f2f0e701a9._comment b/doc/install/Debian/comment_1_029486088d098c2d4f1099f2f0e701a9._comment
new file mode 100644
index 0000000000..9a4ed7c31d
--- /dev/null
+++ b/doc/install/Debian/comment_1_029486088d098c2d4f1099f2f0e701a9._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawla7u6eLKNYZ09Z7xwBffqLaXquMQC07fU"
+ nickname="Matthias"
+ subject="squeeze-backports update?"
+ date="2011-08-17T12:34:46Z"
+ content="""
+Is there going to be an update of git-annex in debian squeeze-backports to a version that supports repository version 3?
+Thx
+"""]]

From a131fabc0dd73a50e4f9a994e876ed1bca608f19 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 17 Aug 2011 15:34:29 +0000
Subject: [PATCH 045/111] Added a comment: Re: squeeze-backports update?

---
 .../comment_2_648e3467e260cdf233acdb0b53313ce0._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/install/Debian/comment_2_648e3467e260cdf233acdb0b53313ce0._comment

diff --git a/doc/install/Debian/comment_2_648e3467e260cdf233acdb0b53313ce0._comment b/doc/install/Debian/comment_2_648e3467e260cdf233acdb0b53313ce0._comment
new file mode 100644
index 0000000000..b8b3d68f33
--- /dev/null
+++ b/doc/install/Debian/comment_2_648e3467e260cdf233acdb0b53313ce0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="Re: squeeze-backports update?"
+ date="2011-08-17T15:34:29Z"
+ content="""
+Yes, I uploaded it last night.
+"""]]

From 3b5f7221305ba6e768711d8326a01b78fb1e4f79 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 13:07:59 -0400
Subject: [PATCH 046/111] init no longer creates .gitattributes

---
 doc/git-annex.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 2865c8af5d..9c202e6689 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -122,7 +122,7 @@ Many git-annex commands will stage changes for later `git commit` by you.
 * init description
 
   Initializes git-annex with a description of the git repository,
-  and sets up `.gitattributes` and the pre-commit hook.
+  and sets up the pre-commit hook.
 
 * describe repository description
 

From 56f6923ccbc6fb1932137b53458a4cece47e69b0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 14:14:43 -0400
Subject: [PATCH 047/111] Now "git annex init" only has to be run once

when a git repository is first being created. Clones will automatically
notice that git-annex is in use and automatically perform a basic
initalization. It's still recommended to run "git annex init" in any
clones, to describe them.
---
 Branch.hs         | 32 ++++++++++++++++------
 CmdLine.hs        | 22 +++++++++++----
 Command/Init.hs   | 38 +++-----------------------
 Command/Uninit.hs | 17 ++----------
 Init.hs           | 69 +++++++++++++++++++++++++++++++++++++++++++++++
 Version.hs        |  6 ++---
 debian/changelog  |  9 +++++++
 7 files changed, 127 insertions(+), 66 deletions(-)
 create mode 100644 Init.hs

diff --git a/Branch.hs b/Branch.hs
index 35e3050936..fd4ce4cede 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -14,6 +14,7 @@ module Branch (
 	files,
 	refExists,
 	hasOrigin,
+	hasSomeBranch,
 	name	
 ) where
 
@@ -124,7 +125,7 @@ getCache file = getState >>= handle
 
 {- Creates the branch, if it does not already exist. -}
 create :: Annex ()
-create = unlessM (refExists fullname) $ do
+create = unlessM hasBranch $ do
 	g <- Annex.gitRepo
 	e <- hasOrigin
 	if e
@@ -154,19 +155,14 @@ update = do
 		 -}
 		staged <- stageJournalFiles
 
-		g <- Annex.gitRepo
-		r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
-		let refs = map (last . words) (lines r)
+		refs <- siblingBranches
 		updated <- catMaybes `liftM` mapM updateRef refs
+		g <- Annex.gitRepo
 		unless (null updated && not staged) $ liftIO $
 			Git.commit g "update" fullname (fullname:updated)
 		Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }
 		invalidateCache
 
-{- Does origin/git-annex exist? -}
-hasOrigin :: Annex Bool
-hasOrigin = refExists originname
-
 {- Checks if a git ref exists. -}
 refExists :: GitRef -> Annex Bool
 refExists ref = do
@@ -174,6 +170,26 @@ refExists ref = do
 	liftIO $ Git.runBool g "show-ref"
 		[Param "--verify", Param "-q", Param ref]
 
+{- Does the main git-annex branch exist? -}
+hasBranch :: Annex Bool
+hasBranch = refExists fullname
+
+{- Does origin/git-annex exist? -}
+hasOrigin :: Annex Bool
+hasOrigin = refExists originname
+
+{- Does the git-annex branch or a foo/git-annex branch exist? -}
+hasSomeBranch :: Annex Bool
+hasSomeBranch = liftM (not . null) siblingBranches
+
+{- List of all git-annex branches, including the main one and any
+ - from remotes. -}
+siblingBranches :: Annex [String]
+siblingBranches = do
+	g <- Annex.gitRepo
+	r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
+	return $ map (last . words) (lines r)
+
 {- Ensures that a given ref has been merged into the index. -}
 updateRef :: GitRef -> Annex (Maybe String)
 updateRef ref
diff --git a/CmdLine.hs b/CmdLine.hs
index c33c497856..ff1758f0dc 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -19,13 +19,14 @@ import Control.Monad (when)
 import qualified Annex
 import qualified AnnexQueue
 import qualified Git
+import qualified Branch
 import Content
 import Types
 import Command
 import Version
 import Options
 import Messages
-import UUID
+import Init
 
 {- Runs the passed command line. -}
 dispatch :: [String] -> [Command] -> [Option] -> String -> Git.Repo -> IO ()
@@ -45,7 +46,7 @@ parseCmd argv header cmds options = do
 		[] -> error $ "unknown command" ++ usagemsg
 		[command] -> do
 			_ <- sequence flags
-			when (cmdusesrepo command) checkVersion
+			checkCmdEnviron command
 			prepCommand command (drop 1 params)
 		_ -> error "internal error: multiple matching commands"
 	where
@@ -57,6 +58,19 @@ parseCmd argv header cmds options = do
 		lookupCmd cmd = filter (\c -> cmd  == cmdname c) cmds
 		usagemsg = "\n\n" ++ usage header cmds options
 
+{- Checks that the command can be run in the current environment. -}
+checkCmdEnviron :: Command -> Annex ()
+checkCmdEnviron command = do
+	when (cmdusesrepo command) $ checkVersion $ do
+		{- Automatically initialize if there is already a git-annex
+		   branch from somewhere. Otherwise, require a manual init
+		   to avoid git-annex accidentially being run in git
+		   repos that did not intend to use it. -}
+		annexed <- Branch.hasSomeBranch
+		if annexed
+			then initialize
+			else error "First run: git-annex init"
+
 {- Usage message with lists of commands and options. -}
 usage :: String -> [Command] -> [Option] -> String
 usage header cmds options =
@@ -95,9 +109,7 @@ tryRun' errnum _ [] = when (errnum > 0) $ error $ show errnum ++ " failed"
 
 {- Actions to perform each time ran. -}
 startup :: Annex Bool
-startup = do
-	prepUUID
-	return True
+startup = return True
 
 {- Cleanup actions. -}
 shutdown :: Annex Bool
diff --git a/Command/Init.hs b/Command/Init.hs
index 71e87050d8..0191060511 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -7,19 +7,13 @@
 
 module Command.Init where
 
-import Control.Monad.State (liftIO)
-import Control.Monad (when, unless)
-import System.Directory
+import Control.Monad (when)
 
 import Command
 import qualified Annex
-import qualified Git
-import qualified Branch
 import UUID
-import Version
 import Messages
-import Types
-import Utility
+import Init
 	
 command :: [Command]
 command = [standaloneCommand "init" paramDesc seek
@@ -39,34 +33,8 @@ start ws = do
 
 perform :: String -> CommandPerform
 perform description = do
-	Branch.create
+	initialize
 	g <- Annex.gitRepo
 	u <- getUUID g
-	setVersion
 	describeUUID u description
-	unless (Git.repoIsLocalBare g) $
-		gitPreCommitHookWrite g
 	next $ return True
-
-{- set up a git pre-commit hook, if one is not already present -}
-gitPreCommitHookWrite :: Git.Repo -> Annex ()
-gitPreCommitHookWrite repo = do
-	exists <- liftIO $ doesFileExist hook
-	if exists
-		then warning $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring"
-		else liftIO $ do
-			viaTmp writeFile hook preCommitScript
-			p <- getPermissions hook
-			setPermissions hook $ p {executable = True}
-	where
-		hook = preCommitHook repo
-
-preCommitHook :: Git.Repo -> FilePath
-preCommitHook repo = 
-	Git.workTree repo ++ "/" ++ Git.gitDir repo ++ "/hooks/pre-commit"
-
-preCommitScript :: String
-preCommitScript = 
-		"#!/bin/sh\n" ++
-		"# automatically configured by git-annex\n" ++ 
-		"git annex pre-commit .\n"
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 8b8d7e364d..195246aa84 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -12,13 +12,11 @@ import System.Directory
 import System.Exit
 
 import Command
-import Messages
-import Types
 import Utility
 import qualified Git
 import qualified Annex
 import qualified Command.Unannex
-import qualified Command.Init
+import Init
 import qualified Branch
 import Content
 import Locations
@@ -47,7 +45,7 @@ perform = next cleanup
 cleanup :: CommandCleanup
 cleanup = do
 	g <- Annex.gitRepo
-	gitPreCommitHookUnWrite g
+	uninitialize
 	mapM_ removeAnnex =<< getKeysPresent
 	liftIO $ removeDirectoryRecursive (gitAnnexDir g)
 	-- avoid normal shutdown
@@ -55,14 +53,3 @@ cleanup = do
 	liftIO $ do
 		Git.run g "branch" [Param "-D", Param Branch.name]
 		exitSuccess
-
-gitPreCommitHookUnWrite :: Git.Repo -> Annex ()
-gitPreCommitHookUnWrite repo = do
-	let hook = Command.Init.preCommitHook repo
-	whenM (liftIO $ doesFileExist hook) $ do
-		c <- liftIO $ readFile hook
-		if c == Command.Init.preCommitScript
-			then liftIO $ removeFile hook
-			else warning $ "pre-commit hook (" ++ hook ++ 
-				") contents modified; not deleting." ++
-				" Edit it to remove call to git annex."
diff --git a/Init.hs b/Init.hs
new file mode 100644
index 0000000000..41256a9530
--- /dev/null
+++ b/Init.hs
@@ -0,0 +1,69 @@
+{- git-annex repository initialization
+ -
+ - Copyright 2010 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Init (initialize, uninitialize) where
+
+import Control.Monad.State (liftIO)
+import Control.Monad (unless)
+import System.Directory
+
+import qualified Annex
+import qualified Git
+import qualified Branch
+import Version
+import Messages
+import Types
+import Utility
+import UUID
+
+initialize :: Annex ()
+initialize = do
+	prepUUID
+	Branch.create
+	setVersion
+	g <- Annex.gitRepo
+	unless (Git.repoIsLocalBare g) $
+		gitPreCommitHookWrite g
+
+uninitialize :: Annex ()
+uninitialize = do
+	g <- Annex.gitRepo
+	gitPreCommitHookUnWrite g
+
+{- set up a git pre-commit hook, if one is not already present -}
+gitPreCommitHookWrite :: Git.Repo -> Annex ()
+gitPreCommitHookWrite repo = do
+	exists <- liftIO $ doesFileExist hook
+	if exists
+		then warning $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring"
+		else liftIO $ do
+			viaTmp writeFile hook preCommitScript
+			p <- getPermissions hook
+			setPermissions hook $ p {executable = True}
+	where
+		hook = preCommitHook repo
+
+gitPreCommitHookUnWrite :: Git.Repo -> Annex ()
+gitPreCommitHookUnWrite repo = do
+	let hook = preCommitHook repo
+	whenM (liftIO $ doesFileExist hook) $ do
+		c <- liftIO $ readFile hook
+		if c == preCommitScript
+			then liftIO $ removeFile hook
+			else warning $ "pre-commit hook (" ++ hook ++ 
+				") contents modified; not deleting." ++
+				" Edit it to remove call to git annex."
+
+preCommitHook :: Git.Repo -> FilePath
+preCommitHook repo = 
+	Git.workTree repo ++ "/" ++ Git.gitDir repo ++ "/hooks/pre-commit"
+
+preCommitScript :: String
+preCommitScript = 
+		"#!/bin/sh\n" ++
+		"# automatically configured by git-annex\n" ++ 
+		"git annex pre-commit .\n"
diff --git a/Version.hs b/Version.hs
index 7e6910fbe4..44fd2e9de1 100644
--- a/Version.hs
+++ b/Version.hs
@@ -39,10 +39,10 @@ getVersion = do
 setVersion :: Annex ()
 setVersion = setConfig versionField defaultVersion
 
-checkVersion :: Annex ()
-checkVersion = getVersion >>= handle
+checkVersion :: Annex () -> Annex ()
+checkVersion initaction = getVersion >>= handle
 	where
-		handle Nothing = error "First run: git-annex init"
+		handle Nothing = initaction
 		handle (Just v) = unless (v `elem` supportedVersions) $
 			error $ "Repository version " ++ v ++ 
 				" is not supported. " ++
diff --git a/debian/changelog b/debian/changelog
index 3eab43578a..9230c00218 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,12 @@
+git-annex (3.20110818) UNRELEASED; urgency=low
+
+  * Now "git annex init" only has to be run once, when a git repository
+    is first being created. Clones will automatically notice that git-annex
+    is in use and automatically perform a basic initalization. It's
+    still recommended to run "git annex init" in any clones, to describe them.
+
+ -- Joey Hess   Wed, 17 Aug 2011 13:44:44 -0400
+
 git-annex (3.20110817) unstable; urgency=low
 
   * Fix shell escaping in rsync special remote.

From cf33eff684de5193379e99745d83c80fd2fb09c0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 14:17:12 -0400
Subject: [PATCH 048/111] git-annex-shell configlist should not be standalone

This makes it initialize the repository with a uuid, and list the uuid,
allowing automatic setup of bare repositories when git-annex is used.
---
 Command/ConfigList.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index d8dbff03af..1b1bb3c34b 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -14,7 +14,7 @@ import Command
 import UUID
 
 command :: [Command]
-command = [standaloneCommand "configlist" paramNothing seek
+command = [repoCommand "configlist" paramNothing seek
 		"outputs relevant git configuration"]
 
 seek :: [CommandSeek]

From 32f27cc3e839a3c243641b953fd4bd0f15dda08a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 14:36:20 -0400
Subject: [PATCH 049/111] when reading configs of local repos, first
 initializeSafe

This auto-generates a uuid if the local repo does not already have one.
---
 CmdLine.hs    | 12 +-----------
 Init.hs       | 17 ++++++++++++++++-
 Remote/Git.hs | 30 +++++++++++++++++-------------
 3 files changed, 34 insertions(+), 25 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index ff1758f0dc..0590f11124 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -19,7 +19,6 @@ import Control.Monad (when)
 import qualified Annex
 import qualified AnnexQueue
 import qualified Git
-import qualified Branch
 import Content
 import Types
 import Command
@@ -60,16 +59,7 @@ parseCmd argv header cmds options = do
 
 {- Checks that the command can be run in the current environment. -}
 checkCmdEnviron :: Command -> Annex ()
-checkCmdEnviron command = do
-	when (cmdusesrepo command) $ checkVersion $ do
-		{- Automatically initialize if there is already a git-annex
-		   branch from somewhere. Otherwise, require a manual init
-		   to avoid git-annex accidentially being run in git
-		   repos that did not intend to use it. -}
-		annexed <- Branch.hasSomeBranch
-		if annexed
-			then initialize
-			else error "First run: git-annex init"
+checkCmdEnviron command = when (cmdusesrepo command) $ checkVersion $ initializeSafe
 
 {- Usage message with lists of commands and options. -}
 usage :: String -> [Command] -> [Option] -> String
diff --git a/Init.hs b/Init.hs
index 41256a9530..36d3ed0fac 100644
--- a/Init.hs
+++ b/Init.hs
@@ -5,7 +5,11 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Init (initialize, uninitialize) where
+module Init (
+	initialize,
+	initializeSafe,
+	uninitialize
+) where
 
 import Control.Monad.State (liftIO)
 import Control.Monad (unless)
@@ -34,6 +38,17 @@ uninitialize = do
 	g <- Annex.gitRepo
 	gitPreCommitHookUnWrite g
 
+{- Call to automatically initialize if there is already a git-annex
+   branch from somewhere. Otherwise, require a manual init
+   to avoid git-annex accidentially being run in git
+   repos that did not intend to use it. -}
+initializeSafe :: Annex ()
+initializeSafe = do
+	annexed <- Branch.hasSomeBranch
+	if annexed
+		then initialize
+		else error "First run: git-annex init"
+
 {- set up a git pre-commit hook, if one is not already present -}
 gitPreCommitHookWrite :: Git.Repo -> Annex ()
 gitPreCommitHookWrite repo = do
diff --git a/Remote/Git.hs b/Remote/Git.hs
index d4847d6105..c588cc73bb 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -28,6 +28,7 @@ import Utility.RsyncFile
 import Remote.Helper.Ssh
 import qualified Remote.Helper.Url as Url
 import Config
+import Init
 
 remote :: RemoteType Annex
 remote = RemoteType {
@@ -79,7 +80,9 @@ tryGitConfigRead r
 	| Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" []
 	| Git.repoIsHttp r = store $ safely $ geturlconfig
 	| Git.repoIsUrl r = return r
-	| otherwise = store $ safely $ Git.configRead r
+	| otherwise = store $ safely $ do
+		onLocal r initializeSafe
+		Git.configRead r
 	where
 		-- Reading config can fail due to IO error or
 		-- for other reasons; catch all possible exceptions.
@@ -124,11 +127,7 @@ inAnnex r key
 	| Git.repoIsUrl r = checkremote
 	| otherwise = safely checklocal
 	where
-		checklocal = do
-			-- run a local check inexpensively,
-			-- by making an Annex monad using the remote
-			a <- Annex.new r
-			Annex.eval a (Content.inAnnex key)
+		checklocal = onLocal r (Content.inAnnex key)
 		checkremote = do
 			showAction $ "checking " ++ Git.repoDescribe r
 			inannex <- onRemote r (boolSystem, False) "inannex" 
@@ -137,6 +136,13 @@ inAnnex r key
 		checkhttp = Url.exists $ keyUrl r key
 		safely a = liftIO (try a ::IO (Either IOException Bool))
 
+{- Runs an action on a local repository inexpensively, by making an annex
+ - monad using that repository. -}
+onLocal :: Git.Repo -> Annex a -> IO a
+onLocal r a = do
+	annex <- Annex.new r
+	Annex.eval annex a
+
 keyUrl :: Git.Repo -> Key -> String
 keyUrl r key = Git.repoLocation r ++ "/" ++ annexLocation key
 
@@ -163,13 +169,11 @@ copyToRemote r key
 		g <- Annex.gitRepo
 		let keysrc = gitAnnexLocation g key
 		-- run copy from perspective of remote
-		liftIO $ do
-			a <- Annex.new r
-			Annex.eval a $ do
-				ok <- Content.getViaTmp key $
-					rsyncOrCopyFile r keysrc
-				Content.saveState
-				return ok
+		liftIO $ onLocal r $ do
+			ok <- Content.getViaTmp key $
+				rsyncOrCopyFile r keysrc
+			Content.saveState
+			return ok
 	| Git.repoIsSsh r = do
 		g <- Annex.gitRepo
 		let keysrc = gitAnnexLocation g key

From 228a724d1d1e1d2f27763bd5f71473acc4d022d9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 14:43:38 -0400
Subject: [PATCH 050/111] improve docs for init

---
 doc/git-annex.mdwn | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 9c202e6689..9d63ca156d 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -121,8 +121,14 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * init description
 
-  Initializes git-annex with a description of the git repository,
-  and sets up the pre-commit hook.
+  Initializes git-annex with a description of the git repository.
+
+  Until a repository (or one of its remotes) has been initialized,
+  git-annex will refuse to operate on it, to avoid accidentially
+  using it in a repository that was not intended to have an annex.
+
+  It's useful, but not mandatory, to initialize each new clone
+  of a repository with its own description.
 
 * describe repository description
 

From b7a4ff1c3185404d36d34b372b016be052394a95 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 18:38:26 -0400
Subject: [PATCH 051/111] optimise initialized check

Avoid running external command if annex.version is set.
---
 CmdLine.hs    |  3 +--
 Init.hs       | 20 ++++++++++++--------
 Remote/Git.hs |  2 +-
 Version.hs    | 19 +++++++------------
 4 files changed, 21 insertions(+), 23 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 0590f11124..2652e5b8fc 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -22,7 +22,6 @@ import qualified Git
 import Content
 import Types
 import Command
-import Version
 import Options
 import Messages
 import Init
@@ -59,7 +58,7 @@ parseCmd argv header cmds options = do
 
 {- Checks that the command can be run in the current environment. -}
 checkCmdEnviron :: Command -> Annex ()
-checkCmdEnviron command = when (cmdusesrepo command) $ checkVersion $ initializeSafe
+checkCmdEnviron command = when (cmdusesrepo command) ensureInitialized
 
 {- Usage message with lists of commands and options. -}
 usage :: String -> [Command] -> [Option] -> String
diff --git a/Init.hs b/Init.hs
index 36d3ed0fac..ae92998bb9 100644
--- a/Init.hs
+++ b/Init.hs
@@ -6,8 +6,8 @@
  -}
 
 module Init (
+	ensureInitialized,
 	initialize,
-	initializeSafe,
 	uninitialize
 ) where
 
@@ -38,16 +38,20 @@ uninitialize = do
 	g <- Annex.gitRepo
 	gitPreCommitHookUnWrite g
 
-{- Call to automatically initialize if there is already a git-annex
+{- Will automatically initialize if there is already a git-annex
    branch from somewhere. Otherwise, require a manual init
    to avoid git-annex accidentially being run in git
    repos that did not intend to use it. -}
-initializeSafe :: Annex ()
-initializeSafe = do
-	annexed <- Branch.hasSomeBranch
-	if annexed
-		then initialize
-		else error "First run: git-annex init"
+ensureInitialized :: Annex ()
+ensureInitialized = do
+	v <- getVersion
+	case v of
+		Just version -> checkVersion version
+		Nothing -> do
+			annexed <- Branch.hasSomeBranch
+			if annexed
+				then initialize
+				else error "First run: git-annex init"
 
 {- set up a git pre-commit hook, if one is not already present -}
 gitPreCommitHookWrite :: Git.Repo -> Annex ()
diff --git a/Remote/Git.hs b/Remote/Git.hs
index c588cc73bb..d8ecd33c4a 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -81,7 +81,7 @@ tryGitConfigRead r
 	| Git.repoIsHttp r = store $ safely $ geturlconfig
 	| Git.repoIsUrl r = return r
 	| otherwise = store $ safely $ do
-		onLocal r initializeSafe
+		onLocal r ensureInitialized
 		Git.configRead r
 	where
 		-- Reading config can fail due to IO error or
diff --git a/Version.hs b/Version.hs
index 44fd2e9de1..fcf6bc4d1d 100644
--- a/Version.hs
+++ b/Version.hs
@@ -7,8 +7,6 @@
 
 module Version where
 
-import Control.Monad (unless)
-
 import Types
 import qualified Annex
 import qualified Git
@@ -39,14 +37,11 @@ getVersion = do
 setVersion :: Annex ()
 setVersion = setConfig versionField defaultVersion
 
-checkVersion :: Annex () -> Annex ()
-checkVersion initaction = getVersion >>= handle
+checkVersion :: Version -> Annex ()
+checkVersion v
+	| v `elem` supportedVersions = return ()
+	| v `elem` upgradableVersions = err "Upgrade this repository: git-annex upgrade"
+	| otherwise = err "Upgrade git-annex."
 	where
-		handle Nothing = initaction
-		handle (Just v) = unless (v `elem` supportedVersions) $
-			error $ "Repository version " ++ v ++ 
-				" is not supported. " ++
-				msg v
-		msg v
-			| v `elem` upgradableVersions = "Upgrade this repository: git-annex upgrade"
-			| otherwise = "Upgrade git-annex."
+		err msg = error $ "Repository version " ++ v ++
+			" is not supported. " ++ msg

From 9e763954ae96201ce577cb51f904bf7756597f14 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 18:42:49 -0400
Subject: [PATCH 052/111] tweak

---
 Init.hs | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/Init.hs b/Init.hs
index ae92998bb9..a11a7c51c9 100644
--- a/Init.hs
+++ b/Init.hs
@@ -1,6 +1,6 @@
 {- git-annex repository initialization
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -43,11 +43,9 @@ uninitialize = do
    to avoid git-annex accidentially being run in git
    repos that did not intend to use it. -}
 ensureInitialized :: Annex ()
-ensureInitialized = do
-	v <- getVersion
-	case v of
-		Just version -> checkVersion version
-		Nothing -> do
+ensureInitialized = getVersion >>= maybe needsinit checkVersion
+	where
+		needsinit = do
 			annexed <- Branch.hasSomeBranch
 			if annexed
 				then initialize

From 0c53ccc675b52d4995b3406d5814116605fe6ec7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 18:52:58 -0400
Subject: [PATCH 053/111] tweak

---
 Init.hs | 32 +++++++++++++++++---------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/Init.hs b/Init.hs
index a11a7c51c9..4708701fa7 100644
--- a/Init.hs
+++ b/Init.hs
@@ -29,14 +29,11 @@ initialize = do
 	prepUUID
 	Branch.create
 	setVersion
-	g <- Annex.gitRepo
-	unless (Git.repoIsLocalBare g) $
-		gitPreCommitHookWrite g
+	gitPreCommitHookWrite
 
 uninitialize :: Annex ()
 uninitialize = do
-	g <- Annex.gitRepo
-	gitPreCommitHookUnWrite g
+	gitPreCommitHookUnWrite
 
 {- Will automatically initialize if there is already a git-annex
    branch from somewhere. Otherwise, require a manual init
@@ -52,8 +49,9 @@ ensureInitialized = getVersion >>= maybe needsinit checkVersion
 				else error "First run: git-annex init"
 
 {- set up a git pre-commit hook, if one is not already present -}
-gitPreCommitHookWrite :: Git.Repo -> Annex ()
-gitPreCommitHookWrite repo = do
+gitPreCommitHookWrite :: Annex ()
+gitPreCommitHookWrite = unlessBare $ do
+	hook <- preCommitHook
 	exists <- liftIO $ doesFileExist hook
 	if exists
 		then warning $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring"
@@ -61,12 +59,10 @@ gitPreCommitHookWrite repo = do
 			viaTmp writeFile hook preCommitScript
 			p <- getPermissions hook
 			setPermissions hook $ p {executable = True}
-	where
-		hook = preCommitHook repo
 
-gitPreCommitHookUnWrite :: Git.Repo -> Annex ()
-gitPreCommitHookUnWrite repo = do
-	let hook = preCommitHook repo
+gitPreCommitHookUnWrite :: Annex ()
+gitPreCommitHookUnWrite = unlessBare $ do
+	hook <- preCommitHook
 	whenM (liftIO $ doesFileExist hook) $ do
 		c <- liftIO $ readFile hook
 		if c == preCommitScript
@@ -75,9 +71,15 @@ gitPreCommitHookUnWrite repo = do
 				") contents modified; not deleting." ++
 				" Edit it to remove call to git annex."
 
-preCommitHook :: Git.Repo -> FilePath
-preCommitHook repo = 
-	Git.workTree repo ++ "/" ++ Git.gitDir repo ++ "/hooks/pre-commit"
+unlessBare :: Annex () -> Annex ()
+unlessBare a = do
+	g <- Annex.gitRepo
+	unless (Git.repoIsLocalBare g) a
+
+preCommitHook :: Annex FilePath
+preCommitHook = do
+	g <- Annex.gitRepo
+	return $ Git.workTree g ++ "/" ++ Git.gitDir g ++ "/hooks/pre-commit"
 
 preCommitScript :: String
 preCommitScript = 

From 8a2197adfab9d8425f92000621a802dce37e124c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 18 Aug 2011 12:20:47 -0400
Subject: [PATCH 054/111] Added annex-cost-command configuration, which can be
 used to vary the cost of a remote based on the output of a shell command.

Also avoided crashing if the user specified cost value cannot be parsed.
---
 Config.hs          | 21 ++++++++++++++++-----
 debian/changelog   |  2 ++
 doc/git-annex.mdwn |  6 ++++++
 3 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/Config.hs b/Config.hs
index 9cbf2d52fd..6f2014483b 100644
--- a/Config.hs
+++ b/Config.hs
@@ -9,6 +9,8 @@ module Config where
 
 import Data.Maybe
 import Control.Monad.State (liftIO)
+import Control.Monad (liftM)
+import System.Cmd.Utils
 
 import qualified Git
 import qualified Annex
@@ -40,14 +42,23 @@ remoteConfig r key = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex
 {- Calculates cost for a remote.
  -
  - The default cost is 100 for local repositories, and 200 for remote
- - repositories; it can also be configured by remote..annex-cost
+ - repositories; it can also be configured by remote..annex-cost,
+ - or if remote..annex-cost-command is set and prints a number, that
+ - is used.
  -}
 remoteCost :: Git.Repo -> Int -> Annex Int
 remoteCost r def = do
-	c <- getConfig r "cost" ""
-	if not $ null c
-		then return $ read c
-		else return def
+	cmd <- getConfig r "cost-command" ""
+	return . safeparse =<< if not $ null cmd
+			then liftM snd $ liftIO $ pipeFrom "sh" ["-c", cmd]
+			else getConfig r "cost" ""
+	where
+		safeparse v
+			| null ws || null ps = def
+			| otherwise = (fst . head) ps
+			where
+				ws = words v
+				ps = reads $ head ws
 
 cheapRemoteCost :: Int
 cheapRemoteCost = 100
diff --git a/debian/changelog b/debian/changelog
index 9230c00218..1ace861a46 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,8 @@ git-annex (3.20110818) UNRELEASED; urgency=low
     is first being created. Clones will automatically notice that git-annex
     is in use and automatically perform a basic initalization. It's
     still recommended to run "git annex init" in any clones, to describe them.
+  * Added annex-cost-command configuration, which can be used to vary the
+    cost of a remote based on the output of a shell command.
 
  -- Joey Hess   Wed, 17 Aug 2011 13:44:44 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 9d63ca156d..a262d465fd 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -424,6 +424,12 @@ Here are all the supported configuration settings.
   The default cost is 100 for local repositories, and 200 for remote
   repositories.
 
+* `remote..annex-cost-command`
+
+  If set, the command is run, and the number it outputs is used as the cost.
+  This allows varying the cost based on eg, the current network. The
+  cost-command can be any shell command line.
+
 * `remote..annex-ignore`
 
   If set to `true`, prevents git-annex

From 7cedd28ab00d93b4a8d70810511a7c28cae65fd1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 18 Aug 2011 12:26:28 -0400
Subject: [PATCH 055/111] tweak

---
 Config.hs | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/Config.hs b/Config.hs
index 6f2014483b..568eb71386 100644
--- a/Config.hs
+++ b/Config.hs
@@ -39,12 +39,9 @@ getConfig r key def = do
 remoteConfig :: Git.Repo -> ConfigKey -> String
 remoteConfig r key = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-" ++ key
 
-{- Calculates cost for a remote.
- -
- - The default cost is 100 for local repositories, and 200 for remote
- - repositories; it can also be configured by remote..annex-cost,
- - or if remote..annex-cost-command is set and prints a number, that
- - is used.
+{- Calculates cost for a remote. Either the default, or as configured 
+ - by remote..annex-cost, or if remote..annex-cost-command
+ - is set and prints a number, that is used.
  -}
 remoteCost :: Git.Repo -> Int -> Annex Int
 remoteCost r def = do

From e97fede8cd86d0eb804ced6d2877f617ba15b1a6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 19 Aug 2011 12:59:07 -0400
Subject: [PATCH 056/111] make gitDir absolute

---
 Git.hs             | 12 ++++++------
 Init.hs            |  2 +-
 git-union-merge.hs |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/Git.hs b/Git.hs
index b226bce0f2..168fe4154e 100644
--- a/Git.hs
+++ b/Git.hs
@@ -247,11 +247,11 @@ attributes repo
 	| configBare repo = workTree repo ++ "/info/.gitattributes"
 	| otherwise = workTree repo ++ "/.gitattributes"
 
-{- Path to a repository's .git directory, relative to its workTree. -}
+{- Path to a repository's .git directory. -}
 gitDir :: Repo -> String
 gitDir repo
-	| configBare repo = ""
-	| otherwise = ".git"
+	| configBare repo = workTree repo
+	| otherwise = workTree repo  ".git"
 
 {- Path to a repository's --work-tree, that is, its top.
  -
@@ -345,10 +345,10 @@ urlAuthPart _ repo = assertUrl repo $ error "internal"
 
 {- Constructs a git command line operating on the specified repo. -}
 gitCommandLine :: Repo -> [CommandParam] -> [CommandParam]
-gitCommandLine repo@(Repo { location = Dir d} ) params =
+gitCommandLine repo@(Repo { location = Dir _ } ) params =
 	-- force use of specified repo via --git-dir and --work-tree
-	[ Param ("--git-dir=" ++ d ++ "/" ++ gitDir repo)
-	, Param ("--work-tree=" ++ d)
+	[ Param ("--git-dir=" ++ gitDir repo)
+	, Param ("--work-tree=" ++ workTree repo)
 	] ++ params
 gitCommandLine repo _ = assertLocal repo $ error "internal"
 
diff --git a/Init.hs b/Init.hs
index 4708701fa7..a469657a11 100644
--- a/Init.hs
+++ b/Init.hs
@@ -79,7 +79,7 @@ unlessBare a = do
 preCommitHook :: Annex FilePath
 preCommitHook = do
 	g <- Annex.gitRepo
-	return $ Git.workTree g ++ "/" ++ Git.gitDir g ++ "/hooks/pre-commit"
+	return $ Git.gitDir g ++ "/hooks/pre-commit"
 
 preCommitScript :: String
 preCommitScript = 
diff --git a/git-union-merge.hs b/git-union-merge.hs
index e763376077..4e1a932b45 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -20,7 +20,7 @@ usage :: IO a
 usage = error $ "bad parameters\n\n" ++ header
 
 tmpIndex :: Git.Repo -> FilePath
-tmpIndex g = Git.workTree g  Git.gitDir g  "index.git-union-merge"
+tmpIndex g = Git.gitDir g  "index.git-union-merge"
 
 setup :: Git.Repo -> IO ()
 setup g = cleanup g -- idempotency

From 021e8e1e0e40543f3861ed8fb0fc12cd2be46d46 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 19 Aug 2011 14:28:07 -0400
Subject: [PATCH 057/111] make Annex an opaque data type

Was a type alias; using newtype has the benefit that type errors will
show "Annex foo" rather than two lines of internal type nonsense. Yay!
There should be no other effects to size or runtime.

I've tried to do this at least twice before (each time I read RWH chapter 10);
finally understood how to this time.. sorta.
---
 Annex.hs | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index f7e3e29f82..fd7e3b391c 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -5,6 +5,8 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
+{-# LANGUAGE GeneralizedNewtypeDeriving #-}
+
 module Annex (
 	Annex,
 	AnnexState(..),
@@ -17,6 +19,7 @@ module Annex (
 ) where
 
 import Control.Monad.State
+import Control.Monad.IO.Control
 
 import qualified Git
 import Git.Queue
@@ -28,7 +31,14 @@ import Types.TrustLevel
 import Types.UUID
 
 -- git-annex's monad
-type Annex = StateT AnnexState IO
+newtype Annex a = Annex { runAnnex :: StateT AnnexState IO a }
+	deriving (
+		Functor,
+		Monad,
+		MonadIO,
+		MonadControlIO,
+		MonadState AnnexState
+	)
 
 -- internal state storage
 data AnnexState = AnnexState
@@ -78,9 +88,9 @@ new gitrepo = newState `liftM` (liftIO . Git.configRead) gitrepo
 
 {- performs an action in the Annex monad -}
 run :: AnnexState -> Annex a -> IO (a, AnnexState)
-run = flip runStateT
+run s a = runStateT (runAnnex a) s
 eval :: AnnexState -> Annex a -> IO a
-eval = flip evalStateT
+eval s a = evalStateT (runAnnex a) s
 
 {- Gets a value from the internal state, selected by the passed value
  - constructor. -}

From e3ca08fad89d6f3b5951da74a8af3631c5ef9604 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 19 Aug 2011 14:36:52 -0400
Subject: [PATCH 058/111] drop an unnecessart liftIO

the liftM on its own can lift all the way into IO.
---
 Annex.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Annex.hs b/Annex.hs
index fd7e3b391c..46dfc9df56 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -84,7 +84,7 @@ newState gitrepo = AnnexState
 
 {- Create and returns an Annex state object for the specified git repo. -}
 new :: Git.Repo -> IO AnnexState
-new gitrepo = newState `liftM` (liftIO . Git.configRead) gitrepo
+new gitrepo = newState `liftM` Git.configRead gitrepo
 
 {- performs an action in the Annex monad -}
 run :: AnnexState -> Annex a -> IO (a, AnnexState)

From 01cd775d9271dfe96917c5557efe93ce0e6f52c4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 19 Aug 2011 20:05:08 -0400
Subject: [PATCH 059/111] Fix broken upgrade from V1 repository. Closes:
 #638584

Had forgotten to keep several old versions of functions needed during this
upgrade.
---
 PresenceLog.hs   |  4 +++-
 Upgrade/V1.hs    | 33 ++++++++++++++++++++++++++-------
 debian/changelog |  1 +
 3 files changed, 30 insertions(+), 8 deletions(-)

diff --git a/PresenceLog.hs b/PresenceLog.hs
index 9c516a8db1..ccb75ff5be 100644
--- a/PresenceLog.hs
+++ b/PresenceLog.hs
@@ -15,10 +15,12 @@ module PresenceLog (
 	LogStatus(..),
 	addLog,
 	readLog,
+	parseLog,
 	writeLog,
 	logNow,
 	compactLog,
-	currentLog
+	currentLog,
+	LogLine
 ) where
 
 import Data.Time.Clock.POSIX
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index c41310880f..160d9309fe 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -22,7 +22,7 @@ import Types.Key
 import Content
 import Types
 import Locations
-import LocationLog
+import PresenceLog
 import qualified Annex
 import qualified AnnexQueue
 import qualified Git
@@ -123,7 +123,7 @@ moveLocationLogs = do
 					else return []
 			move (l, k) = do
 				g <- Annex.gitRepo
-				let dest = logFile k
+				let dest = logFile2 g k
 				let dir = Upgrade.V2.gitStateDir g
 				let f = dir  l
 				liftIO $ createDirectoryIfMissing True (parentDir dest)
@@ -131,9 +131,9 @@ moveLocationLogs = do
 				-- log files that are not checked into git,
 				-- as well as merging with already upgraded
 				-- logs that have been pulled from elsewhere
-				old <- readLog f
-				new <- readLog dest
-				writeLog dest (old++new)
+				old <- liftIO $ readLog1 f
+				new <- liftIO $ readLog1 dest
+				liftIO $ writeLog1 dest (old++new)
 				AnnexQueue.add "add" [Param "--"] [dest]
 				AnnexQueue.add "add" [Param "--"] [f]
 				AnnexQueue.add "rm" [Param "--quiet", Param "-f", Param "--"] [f]
@@ -186,8 +186,11 @@ fileKey1 :: FilePath -> Key
 fileKey1 file = readKey1 $
 	replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file
 
-logFile1 :: Git.Repo -> Key -> String
-logFile1 repo key = Upgrade.V2.gitStateDir repo ++ keyFile1 key ++ ".log"
+writeLog1 :: FilePath -> [LogLine] -> IO ()
+writeLog1 file ls = viaTmp writeFile file (unlines $ map show ls)
+
+readLog1 :: FilePath -> IO [LogLine]
+readLog1 file = catch (return . parseLog =<< readFileStrict file) (const $ return [])
 
 lookupFile1 :: FilePath -> Annex (Maybe (Key, Backend Annex))
 lookupFile1 file = do
@@ -230,3 +233,19 @@ getKeyFilesPresent1' dir = do
 			case result of
 				Right s -> return $ isRegularFile s
 				Left _ -> return False
+
+logFile1 :: Git.Repo -> Key -> String
+logFile1 repo key = Upgrade.V2.gitStateDir repo ++ keyFile1 key ++ ".log"
+
+logFile2 :: Git.Repo -> Key -> String
+logFile2 = logFile' hashDirLower
+
+logFile' :: (Key -> FilePath) -> Git.Repo -> Key -> String
+logFile' hasher repo key =
+	gitStateDir repo ++ hasher key ++ keyFile key ++ ".log"
+
+stateDir :: FilePath
+stateDir = addTrailingPathSeparator $ ".git-annex"
+
+gitStateDir :: Git.Repo -> FilePath
+gitStateDir repo = addTrailingPathSeparator $ Git.workTree repo  stateDir
diff --git a/debian/changelog b/debian/changelog
index 1ace861a46..7d3315103e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,7 @@ git-annex (3.20110818) UNRELEASED; urgency=low
     still recommended to run "git annex init" in any clones, to describe them.
   * Added annex-cost-command configuration, which can be used to vary the
     cost of a remote based on the output of a shell command.
+  * Fix broken upgrade from V1 repository. Closes: #638584
 
  -- Joey Hess   Wed, 17 Aug 2011 13:44:44 -0400
 

From 3786f8d3487fe5a460d40045d3bcaaa69f3f84d7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 19 Aug 2011 20:38:36 -0400
Subject: [PATCH 060/111] releasing version 3.20110819

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 7d3315103e..a8280a156d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20110818) UNRELEASED; urgency=low
+git-annex (3.20110819) unstable; urgency=low
 
   * Now "git annex init" only has to be run once, when a git repository
     is first being created. Clones will automatically notice that git-annex
@@ -8,7 +8,7 @@ git-annex (3.20110818) UNRELEASED; urgency=low
     cost of a remote based on the output of a shell command.
   * Fix broken upgrade from V1 repository. Closes: #638584
 
- -- Joey Hess   Wed, 17 Aug 2011 13:44:44 -0400
+ -- Joey Hess   Fri, 19 Aug 2011 20:34:09 -0400
 
 git-annex (3.20110817) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 14188cd350..a37c187c90 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110817
+Version: 3.20110819
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 71215263855e2d5e940bb0086efd7691aa0139d8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 19 Aug 2011 20:39:03 -0400
Subject: [PATCH 061/111] add news item for git-annex 3.20110819

---
 doc/news/version_3.20110702.mdwn | 22 ----------------------
 doc/news/version_3.20110819.mdwn |  9 +++++++++
 2 files changed, 9 insertions(+), 22 deletions(-)
 delete mode 100644 doc/news/version_3.20110702.mdwn
 create mode 100644 doc/news/version_3.20110819.mdwn

diff --git a/doc/news/version_3.20110702.mdwn b/doc/news/version_3.20110702.mdwn
deleted file mode 100644
index a5bb925552..0000000000
--- a/doc/news/version_3.20110702.mdwn
+++ /dev/null
@@ -1,22 +0,0 @@
-News for git-annex 3.20110702:
-
-The URL backend has been removed. Instead the new web remote can be used.
-
-git-annex 3.20110702 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Now the web can be used as a special remote.
-     This feature replaces the old URL backend.
-   * addurl: New command to download an url and store it in the annex.
-   * Sped back up fsck, copy --from, and other commands that often
-     have to read a lot of information from the git-annex branch. Such
-     commands are now faster than they were before introduction of the
-     git-annex branch.
-   * Always ensure git-annex branch exists.
-   * Modify location log parser to allow future expansion.
-   * --force will cause add, etc, to operate on ignored files.
-   * Avoid mangling encoding when storing the description of repository
-     and other content.
-   * cabal can now be used to build git-annex. This is substantially
-     slower than using make, does not build or install documentation,
-     does not run the test suite, and is not particularly recommended,
-     but could be useful to some."""]]
diff --git a/doc/news/version_3.20110819.mdwn b/doc/news/version_3.20110819.mdwn
new file mode 100644
index 0000000000..bbc6abdbc7
--- /dev/null
+++ b/doc/news/version_3.20110819.mdwn
@@ -0,0 +1,9 @@
+git-annex 3.20110819 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Now "git annex init" only has to be run once, when a git repository
+     is first being created. Clones will automatically notice that git-annex
+     is in use and automatically perform a basic initalization. It's
+     still recommended to run "git annex init" in any clones, to describe them.
+   * Added annex-cost-command configuration, which can be used to vary the
+     cost of a remote based on the output of a shell command.
+   * Fix broken upgrade from V1 repository. Closes: #[638584](http://bugs.debian.org/638584)"""]]
\ No newline at end of file

From ec746c511f5666fc214eba1a477d1ababfe9d367 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 20 Aug 2011 12:52:29 -0400
Subject: [PATCH 062/111] note about why curl -# is used

I'd rather use wget really, but as git-annex uses libcurl elsewhere, it
seems best to stick with curl. And making this configurable seems
overboard.
---
 Remote/Helper/Url.hs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/Remote/Helper/Url.hs b/Remote/Helper/Url.hs
index d3aea56220..af1fee8f04 100644
--- a/Remote/Helper/Url.hs
+++ b/Remote/Helper/Url.hs
@@ -39,6 +39,10 @@ exists url =
 download :: URLString -> FilePath -> Annex Bool
 download url file = do
 	showOutput -- make way for curl progress bar
+	-- Uses the -# progress display, because the normal one is very
+	-- confusing when resuming, showing the remainder to download
+	-- as the whole file, and not indicating how much percent was
+	-- downloaded before the resume.
 	liftIO $ boolSystem "curl" [Params "-L -C - -# -o", File file, File url]
 
 {- Downloads a small file. -}

From 737b5d14c91101d46e20999e33461e9059dd9f28 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 20 Aug 2011 16:11:42 -0400
Subject: [PATCH 063/111] moved files around

---
 .gitignore                           |  2 +-
 Backend/SHA.hs                       |  2 +-
 TestConfig.hs => Build/TestConfig.hs |  8 ++++----
 Command/Add.hs                       |  2 +-
 Command/AddUrl.hs                    |  4 ++--
 Command/Map.hs                       |  2 +-
 Command/Version.hs                   |  2 +-
 Content.hs                           |  2 +-
 Makefile                             | 10 ++++------
 Remote/Bup.hs                        |  2 +-
 Remote/Git.hs                        |  4 ++--
 Remote/Web.hs                        |  2 +-
 UUID.hs                              |  2 +-
 Utility/CopyFile.hs                  |  2 +-
 {Remote/Helper => Utility}/Ssh.hs    |  2 +-
 StatFS.hsc => Utility/StatFS.hsc     |  2 +-
 Touch.hsc => Utility/Touch.hsc       |  2 +-
 {Remote/Helper => Utility}/Url.hs    |  4 ++--
 configure.hs                         |  2 +-
 19 files changed, 28 insertions(+), 30 deletions(-)
 rename TestConfig.hs => Build/TestConfig.hs (94%)
 rename {Remote/Helper => Utility}/Ssh.hs (98%)
 rename StatFS.hsc => Utility/StatFS.hsc (98%)
 rename Touch.hsc => Utility/Touch.hsc (99%)
 rename {Remote/Helper => Utility}/Url.hs (96%)

diff --git a/.gitignore b/.gitignore
index d5bf54c813..1fdad216dc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,7 +2,7 @@
 *.o
 test
 configure
-SysConfig.hs
+Build/SysConfig.hs
 git-annex
 git-annex-shell
 git-union-merge
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index bae19be003..b8a0d254b7 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -24,7 +24,7 @@ import Types
 import Types.Backend
 import Types.Key
 import Utility
-import qualified SysConfig
+import qualified Build.SysConfig as SysConfig
 
 type SHASize = Int
 
diff --git a/TestConfig.hs b/Build/TestConfig.hs
similarity index 94%
rename from TestConfig.hs
rename to Build/TestConfig.hs
index 8cfae7f0c7..e8a0d13368 100644
--- a/TestConfig.hs
+++ b/Build/TestConfig.hs
@@ -1,6 +1,6 @@
-{- Tests the system and generates SysConfig.hs. -}
+{- Tests the system and generates Build.SysConfig.hs. -}
 
-module TestConfig where
+module Build.TestConfig where
 
 import System.IO
 import System.Cmd
@@ -33,12 +33,12 @@ instance Show Config where
 			valuetype (MaybeStringConfig _) = "Maybe String"
 
 writeSysConfig :: [Config] -> IO ()
-writeSysConfig config = writeFile "SysConfig.hs" body
+writeSysConfig config = writeFile "Build/SysConfig.hs" body
 	where
 		body = unlines $ header ++ map show config ++ footer
 		header = [
 			  "{- Automatically generated. -}"
-			, "module SysConfig where"
+			, "module Build.SysConfig where"
 			, ""
 			]
 		footer = []
diff --git a/Command/Add.hs b/Command/Add.hs
index d8947fb07c..407a36093f 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -24,7 +24,7 @@ import Types
 import Content
 import Messages
 import Utility
-import Touch
+import Utility.Touch
 import Locations
 
 command :: [Command]
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index add71c8209..72a6b671dd 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -14,7 +14,7 @@ import System.Directory
 
 import Command
 import qualified Backend
-import qualified Remote.Helper.Url
+import qualified Utility.Url as Url
 import qualified Remote.Web
 import qualified Command.Add
 import qualified Annex
@@ -53,7 +53,7 @@ download url file = do
 	let dummykey = Backend.URL.fromUrl url
 	let tmp = gitAnnexTmpLocation g dummykey
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
-	ok <- Remote.Helper.Url.download url tmp
+	ok <- Url.download url tmp
 	if ok
 		then do
 			[(_, backend)] <- Backend.chooseBackends [file]
diff --git a/Command/Map.hs b/Command/Map.hs
index 75c5b0b550..3fd6e42a10 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -22,7 +22,7 @@ import Types
 import Utility
 import UUID
 import Trust
-import Remote.Helper.Ssh
+import Utility.Ssh
 import qualified Utility.Dot as Dot
 
 -- a link from the first repository to the second (its remote)
diff --git a/Command/Version.hs b/Command/Version.hs
index 2392c9bf6b..1ff829a22a 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -12,7 +12,7 @@ import Data.String.Utils
 import Data.Maybe
 
 import Command
-import qualified SysConfig
+import qualified Build.SysConfig as SysConfig
 import Version
 
 command :: [Command]
diff --git a/Content.hs b/Content.hs
index c63042dfb5..e89f4c45af 100644
--- a/Content.hs
+++ b/Content.hs
@@ -41,7 +41,7 @@ import qualified Annex
 import qualified AnnexQueue
 import qualified Branch
 import Utility
-import StatFS
+import Utility.StatFS
 import Types.Key
 import Utility.DataUnits
 import Config
diff --git a/Makefile b/Makefile
index ccf98f6252..fa087c1ae2 100644
--- a/Makefile
+++ b/Makefile
@@ -8,12 +8,11 @@ GHCMAKE=ghc $(GHCFLAGS) --make
 
 bins=git-annex git-annex-shell git-union-merge
 mans=git-annex.1 git-annex-shell.1 git-union-merge.1
+sources=Build/SysConfig.hs Utility/StatFS.hs Utility/Touch.hs Remote/S3.hs
 
 all: $(bins) $(mans) docs
 
-sources: SysConfig.hs StatFS.hs Touch.hs Remote/S3.hs
-
-SysConfig.hs: configure.hs TestConfig.hs
+Build/SysConfig.hs: configure.hs Build/TestConfig.hs
 	$(GHCMAKE) configure
 	./configure
 
@@ -30,7 +29,7 @@ Remote/S3.o: Remote/S3.hs
 		echo "** building without S3 support"; \
 	fi
 
-$(bins): SysConfig.hs Touch.hs StatFS.hs Remote/S3.o
+$(bins): $(sources)
 	$(GHCMAKE) $@
 
 git-annex.1: doc/git-annex.mdwn
@@ -82,8 +81,7 @@ docs: $(mans)
 		--exclude='news/.*'
 
 clean:
-	rm -rf build $(bins) $(mans) test configure  *.tix .hpc \
-		StatFS.hs Touch.hs SysConfig.hs Remote/S3.hs
+	rm -rf build $(bins) $(mans) test configure  *.tix .hpc $(sources)
 	rm -rf doc/.ikiwiki html dist
 	find . \( -name \*.o -or -name \*.hi \) -exec rm {} \;
 
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index c82f84745d..069209792d 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -30,7 +30,7 @@ import Locations
 import Config
 import Utility
 import Messages
-import Remote.Helper.Ssh
+import Utility.Ssh
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
diff --git a/Remote/Git.hs b/Remote/Git.hs
index d8ecd33c4a..80ba8a153d 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -25,8 +25,8 @@ import qualified Content
 import Messages
 import Utility.CopyFile
 import Utility.RsyncFile
-import Remote.Helper.Ssh
-import qualified Remote.Helper.Url as Url
+import Utility.Ssh
+import qualified Utility.Url as Url
 import Config
 import Init
 
diff --git a/Remote/Web.hs b/Remote/Web.hs
index cc96d5306d..5bc6a204b9 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -24,7 +24,7 @@ import Config
 import PresenceLog
 import LocationLog
 import Locations
-import qualified Remote.Helper.Url as Url
+import qualified Utility.Url as Url
 
 type URLString = String
 
diff --git a/UUID.hs b/UUID.hs
index cfa519baa3..fa71bed391 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -33,7 +33,7 @@ import qualified Branch
 import Types
 import Types.UUID
 import qualified Annex
-import qualified SysConfig
+import qualified Build.SysConfig as SysConfig
 import Config
 
 configkey :: String
diff --git a/Utility/CopyFile.hs b/Utility/CopyFile.hs
index 2e06dd92bb..befb00f8f2 100644
--- a/Utility/CopyFile.hs
+++ b/Utility/CopyFile.hs
@@ -10,7 +10,7 @@ module Utility.CopyFile (copyFile) where
 import System.Directory (doesFileExist, removeFile)
 
 import Utility
-import qualified SysConfig
+import qualified Build.SysConfig as SysConfig
 
 {- The cp command is used, because I hate reinventing the wheel,
  - and because this allows easy access to features like cp --reflink. -}
diff --git a/Remote/Helper/Ssh.hs b/Utility/Ssh.hs
similarity index 98%
rename from Remote/Helper/Ssh.hs
rename to Utility/Ssh.hs
index 478b018812..6cbc362a03 100644
--- a/Remote/Helper/Ssh.hs
+++ b/Utility/Ssh.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Remote.Helper.Ssh where
+module Utility.Ssh where
 
 import Control.Monad.State (liftIO)
 
diff --git a/StatFS.hsc b/Utility/StatFS.hsc
similarity index 98%
rename from StatFS.hsc
rename to Utility/StatFS.hsc
index feb82aa9af..d3e4a507e5 100644
--- a/StatFS.hsc
+++ b/Utility/StatFS.hsc
@@ -45,7 +45,7 @@
 {-# LANGUAGE CPP, ForeignFunctionInterface, EmptyDataDecls #-}
 
 
-module StatFS ( FileSystemStats(..), getFileSystemStats ) where
+module Utility.StatFS ( FileSystemStats(..), getFileSystemStats ) where
 
 import Foreign
 import Foreign.C.Types
diff --git a/Touch.hsc b/Utility/Touch.hsc
similarity index 99%
rename from Touch.hsc
rename to Utility/Touch.hsc
index dd0c38984e..f27ac31360 100644
--- a/Touch.hsc
+++ b/Utility/Touch.hsc
@@ -7,7 +7,7 @@
 
 {-# LANGUAGE ForeignFunctionInterface #-}
 
-module Touch (
+module Utility.Touch (
 	TimeSpec(..),
 	touchBoth,
 	touch
diff --git a/Remote/Helper/Url.hs b/Utility/Url.hs
similarity index 96%
rename from Remote/Helper/Url.hs
rename to Utility/Url.hs
index af1fee8f04..5954e0ff7e 100644
--- a/Remote/Helper/Url.hs
+++ b/Utility/Url.hs
@@ -1,11 +1,11 @@
-{- Url downloading for remotes.
+{- Url downloading.
  -
  - Copyright 2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Remote.Helper.Url (
+module Utility.Url (
 	exists,
 	download,
 	get
diff --git a/configure.hs b/configure.hs
index bfdfa32dd5..9f7179c539 100644
--- a/configure.hs
+++ b/configure.hs
@@ -3,7 +3,7 @@
 import System.Directory
 import Data.List
 
-import TestConfig
+import Build.TestConfig
 
 tests :: [TestCase]
 tests =

From 1c25df3c1b19ad8406cbe2aef9dfd215ef9f5ce8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 21 Aug 2011 12:59:49 -0400
Subject: [PATCH 064/111] outdated comment

---
 AnnexQueue.hs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/AnnexQueue.hs b/AnnexQueue.hs
index 79116c48af..56d71f3238 100644
--- a/AnnexQueue.hs
+++ b/AnnexQueue.hs
@@ -19,8 +19,7 @@ import Messages
 import qualified Git.Queue
 import Utility
 
-{- Adds a git command to the queue, possibly running previously queued
- - actions if enough have accumulated. -}
+{- Adds a git command to the queue. -}
 add :: String -> [CommandParam] -> [FilePath] -> Annex ()
 add command params files = do
 	q <- getState repoqueue

From 06ce5741365ae4602ca64204729e6a23705059ee Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 21 Aug 2011 13:17:12 -0400
Subject: [PATCH 065/111] tweak

---
 Config.hs  | 5 ++---
 Utility.hs | 6 +++---
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/Config.hs b/Config.hs
index 568eb71386..493a907009 100644
--- a/Config.hs
+++ b/Config.hs
@@ -51,11 +51,10 @@ remoteCost r def = do
 			else getConfig r "cost" ""
 	where
 		safeparse v
-			| null ws || null ps = def
-			| otherwise = (fst . head) ps
+			| null ws = def
+			| otherwise = fromMaybe def $ readMaybe $ head ws
 			where
 				ws = words v
-				ps = reads $ head ws
 
 cheapRemoteCost :: Int
 cheapRemoteCost = 100
diff --git a/Utility.hs b/Utility.hs
index 511350898a..8a332601b7 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -75,7 +75,7 @@ toCommand = (>>= unwrap)
 		unwrap (Params s) = filter (not . null) (split " " s)
 		-- Files that start with a dash are modified to avoid
 		-- the command interpreting them as options.
-		unwrap (File ('-':s)) = ["./-" ++ s]
+		unwrap (File s@('-':_)) = ["./" ++ s]
 		unwrap (File s) = [s]
 
 {- Run a system command, and returns True or False
@@ -257,7 +257,7 @@ viaTmp a file content = do
 
 {- Runs an action with a temp file, then removes the file. -}
 withTempFile :: String -> (FilePath -> Handle -> IO a) -> IO a
-withTempFile template action = bracket create remove use
+withTempFile template a = bracket create remove use
 	where
 		create = do
 			tmpdir <- catch getTemporaryDirectory (const $ return ".")
@@ -265,7 +265,7 @@ withTempFile template action = bracket create remove use
 		remove (name, handle) = do
 			hClose handle
 			catchBool (removeFile name >> return True)
-		use (name, handle) = action name handle
+		use (name, handle) = a name handle
 
 {- Lists the contents of a directory.
  - Unlike getDirectoryContents, paths are not relative to the directory. -}

From 06f509854af4e019434ddca73309fcd2a5b47463 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 21 Aug 2011 13:19:33 -0400
Subject: [PATCH 066/111] file moved

---
 debian/copyright | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/copyright b/debian/copyright
index 112ad54aaa..a8a38913e4 100644
--- a/debian/copyright
+++ b/debian/copyright
@@ -8,7 +8,7 @@ License: GPL-3+
  this package's source, or in /usr/share/common-licenses/GPL-3 on
  Debian systems.
 
-Files: StatFS.hsc
+Files: Utility/StatFS.hsc
 Copyright: Jose A Ortega Ruiz 
 License: BSD-3-clause
  -- All rights reserved.

From 4c73d77b42e97ad740d5731ad73c40a31c0c84f9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 21 Aug 2011 14:59:34 -0400
Subject: [PATCH 067/111] avoid the functor

fmap = liftM
---
 Annex.hs  | 1 -
 Branch.hs | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 46dfc9df56..07316bd370 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -33,7 +33,6 @@ import Types.UUID
 -- git-annex's monad
 newtype Annex a = Annex { runAnnex :: StateT AnnexState IO a }
 	deriving (
-		Functor,
 		Monad,
 		MonadIO,
 		MonadControlIO,
diff --git a/Branch.hs b/Branch.hs
index fd4ce4cede..6abf6d9c2c 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -321,7 +321,7 @@ getJournalFile file = do
 
 {- List of journal files. -}
 getJournalFiles :: Annex [FilePath]
-getJournalFiles = fmap (map fileJournal) getJournalFilesRaw
+getJournalFiles = liftM (map fileJournal) getJournalFilesRaw
 
 getJournalFilesRaw :: Annex [FilePath]
 getJournalFilesRaw = do

From 203148363f459635b1be40b8c6ed376073230dda Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 22 Aug 2011 16:14:12 -0400
Subject: [PATCH 068/111] split groups of related functions out of Utility

---
 .gitignore               |   4 +-
 AnnexQueue.hs            |   2 +-
 Backend/SHA.hs           |   2 +-
 Branch.hs                |   2 +
 Command/Add.hs           |   3 +-
 Command/AddUrl.hs        |   2 +-
 Command/Drop.hs          |   1 +
 Command/DropUnused.hs    |   2 +-
 Command/Find.hs          |   2 +-
 Command/Fix.hs           |   3 +-
 Command/FromKey.hs       |   3 +-
 Command/Fsck.hs          |   1 +
 Command/Lock.hs          |   2 +-
 Command/Map.hs           |   2 +-
 Command/Migrate.hs       |   2 +-
 Command/RecvKey.hs       |   2 +-
 Command/SendKey.hs       |   2 +-
 Command/SetKey.hs        |   2 +-
 Command/Unannex.hs       |   3 +-
 Command/Uninit.hs        |   2 +-
 Command/Unlock.hs        |   3 +-
 Config.hs                |   1 +
 Content.hs               |   2 +
 Crypto.hs                |   1 +
 Git.hs                   |   3 +
 Git/LsFiles.hs           |   2 +-
 Git/Queue.hs             |   8 +-
 Git/UnionMerge.hs        |   2 +-
 Init.hs                  |   1 +
 Remote/Bup.hs            |   2 +
 Remote/Directory.hs      |   2 +
 Remote/Git.hs            |   2 +
 Remote/Helper/Special.hs |   2 +-
 Remote/Hook.hs           |   1 +
 Remote/Rsync.hs          |   3 +
 Upgrade/V1.hs            |   2 +
 Upgrade/V2.hs            |   2 +
 Utility.hs               | 215 +--------------------------------------
 Utility/Conditional.hs   |  26 +++++
 Utility/CopyFile.hs      |   3 +-
 Utility/Path.hs          |  92 +++++++++++++++++
 Utility/RsyncFile.hs     |   2 +-
 Utility/SafeCommand.hs   | 104 +++++++++++++++++++
 Utility/Ssh.hs           |   2 +-
 Utility/Url.hs           |   2 +-
 git-annex-shell.hs       |   3 +-
 test.hs                  |  45 ++++----
 47 files changed, 312 insertions(+), 265 deletions(-)
 create mode 100644 Utility/Conditional.hs
 create mode 100644 Utility/Path.hs
 create mode 100644 Utility/SafeCommand.hs

diff --git a/.gitignore b/.gitignore
index 1fdad216dc..7d2504de6f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,7 +13,7 @@ doc/.ikiwiki
 html
 *.tix
 .hpc
-Touch.hs
-StatFS.hs
+Utility/Touch.hs
+Utility/StatFS.hs
 Remote/S3.hs
 dist
diff --git a/AnnexQueue.hs b/AnnexQueue.hs
index 56d71f3238..d155a7b81f 100644
--- a/AnnexQueue.hs
+++ b/AnnexQueue.hs
@@ -17,7 +17,7 @@ import Control.Monad (when, unless)
 import Annex
 import Messages
 import qualified Git.Queue
-import Utility
+import Utility.SafeCommand
 
 {- Adds a git command to the queue. -}
 add :: String -> [CommandParam] -> [FilePath] -> Annex ()
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index b8a0d254b7..ed2a47db9b 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -23,7 +23,7 @@ import Content
 import Types
 import Types.Backend
 import Types.Key
-import Utility
+import Utility.SafeCommand
 import qualified Build.SysConfig as SysConfig
 
 type SHASize = Int
diff --git a/Branch.hs b/Branch.hs
index 6abf6d9c2c..d5bfe1b09f 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -37,6 +37,8 @@ import qualified Git
 import qualified Git.UnionMerge
 import qualified Annex
 import Utility
+import Utility.Conditional
+import Utility.SafeCommand
 import Types
 import Messages
 import Locations
diff --git a/Command/Add.hs b/Command/Add.hs
index 407a36093f..579c4171b1 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -23,8 +23,9 @@ import LocationLog
 import Types
 import Content
 import Messages
-import Utility
+import Utility.Conditional
 import Utility.Touch
+import Utility.SafeCommand
 import Locations
 
 command :: [Command]
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 72a6b671dd..55e51100ce 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -23,7 +23,7 @@ import Messages
 import Content
 import PresenceLog
 import Locations
-import Utility
+import Utility.Path
 
 command :: [Command]
 command = [repoCommand "addurl" paramPath seek "add urls to annex"]
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 14f098349e..6e688d6632 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -15,6 +15,7 @@ import Types
 import Content
 import Messages
 import Utility
+import Utility.Conditional
 import Trust
 import Config
 
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 41bcd6aa78..4ad2aa85bb 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -22,7 +22,7 @@ import qualified Command.Move
 import qualified Remote
 import qualified Git
 import Types.Key
-import Utility
+import Utility.Conditional
 
 type UnusedMap = M.Map String Key
 
diff --git a/Command/Find.hs b/Command/Find.hs
index 9d760ff5a8..0716c5297e 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -11,7 +11,7 @@ import Control.Monad.State (liftIO)
 
 import Command
 import Content
-import Utility
+import Utility.Conditional
 
 command :: [Command]
 command = [repoCommand "find" (paramOptional $ paramRepeating paramPath) seek
diff --git a/Command/Fix.hs b/Command/Fix.hs
index 47b0c4c9a0..b24f8e33c2 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -13,7 +13,8 @@ import System.Directory
 
 import Command
 import qualified AnnexQueue
-import Utility
+import Utility.Path
+import Utility.SafeCommand
 import Content
 import Messages
 
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index d59f1de397..89c3f4e912 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -14,10 +14,11 @@ import Control.Monad (unless)
 
 import Command
 import qualified AnnexQueue
-import Utility
+import Utility.SafeCommand
 import Content
 import Messages
 import Types.Key
+import Utility.Path
 
 command :: [Command]
 command = [repoCommand "fromkey" paramPath seek
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 0d3ecb58f1..6ccec05fb3 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -27,6 +27,7 @@ import LocationLog
 import Locations
 import Trust
 import Utility.DataUnits
+import Utility.Path
 import Config
 
 command :: [Command]
diff --git a/Command/Lock.hs b/Command/Lock.hs
index d39df5f335..77d1ff94f9 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -13,7 +13,7 @@ import System.Directory
 import Command
 import Messages
 import qualified AnnexQueue
-import Utility
+import Utility.SafeCommand
 	
 command :: [Command]
 command = [repoCommand "lock" paramPath seek "undo unlock command"]
diff --git a/Command/Map.hs b/Command/Map.hs
index 3fd6e42a10..ef8e04d909 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -19,7 +19,7 @@ import qualified Annex
 import qualified Git
 import Messages
 import Types
-import Utility
+import Utility.SafeCommand
 import UUID
 import Trust
 import Utility.Ssh
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 5ae8354406..25227ae162 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -20,7 +20,7 @@ import Locations
 import Types
 import Content
 import Messages
-import Utility
+import Utility.Conditional
 import qualified Command.Add
 
 command :: [Command]
diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index e2f7c74abb..be6163558f 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -13,8 +13,8 @@ import System.Exit
 import Command
 import CmdLine
 import Content
-import Utility
 import Utility.RsyncFile
+import Utility.Conditional
 
 command :: [Command]
 command = [repoCommand "recvkey" paramKey seek
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index 02fedb349e..f676ae947a 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -14,8 +14,8 @@ import Locations
 import qualified Annex
 import Command
 import Content
-import Utility
 import Utility.RsyncFile
+import Utility.Conditional
 import Messages
 
 command :: [Command]
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index 807cbd5b93..2f6f9ea9ee 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -10,7 +10,7 @@ module Command.SetKey where
 import Control.Monad.State (liftIO)
 
 import Command
-import Utility
+import Utility.SafeCommand
 import LocationLog
 import Content
 import Messages
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 960f99722f..54ef2fc68e 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -16,7 +16,8 @@ import Command
 import qualified Command.Drop
 import qualified Annex
 import qualified AnnexQueue
-import Utility
+import Utility.SafeCommand
+import Utility.Path
 import LocationLog
 import Types
 import Content
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 195246aa84..fadae0e5a9 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -12,7 +12,7 @@ import System.Directory
 import System.Exit
 
 import Command
-import Utility
+import Utility.SafeCommand
 import qualified Git
 import qualified Annex
 import qualified Command.Unannex
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index 280eff9de7..0daf1b3218 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -16,8 +16,9 @@ import Types
 import Messages
 import Locations
 import Content
+import Utility.Conditional
 import Utility.CopyFile
-import Utility
+import Utility.Path
 
 command :: [Command]
 command =
diff --git a/Config.hs b/Config.hs
index 493a907009..12f6480470 100644
--- a/Config.hs
+++ b/Config.hs
@@ -16,6 +16,7 @@ import qualified Git
 import qualified Annex
 import Types
 import Utility
+import Utility.SafeCommand
 
 type ConfigKey = String
 
diff --git a/Content.hs b/Content.hs
index e89f4c45af..ba99f1330d 100644
--- a/Content.hs
+++ b/Content.hs
@@ -41,7 +41,9 @@ import qualified Annex
 import qualified AnnexQueue
 import qualified Branch
 import Utility
+import Utility.Conditional
 import Utility.StatFS
+import Utility.Path
 import Types.Key
 import Utility.DataUnits
 import Config
diff --git a/Crypto.hs b/Crypto.hs
index 4fc41ede04..ed29747aa6 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -48,6 +48,7 @@ import Types.Key
 import Types.Remote
 import Utility
 import Utility.Base64
+import Utility.SafeCommand
 import Types.Crypto
 
 {- The first half of a Cipher is used for HMAC; the remainder
diff --git a/Git.hs b/Git.hs
index 168fe4154e..7155b26343 100644
--- a/Git.hs
+++ b/Git.hs
@@ -85,6 +85,9 @@ import System.Exit
 import System.Posix.Env (setEnv, unsetEnv, getEnv)
 
 import Utility
+import Utility.Path
+import Utility.Conditional
+import Utility.SafeCommand
 
 {- There are two types of repositories; those on local disk and those
  - accessed via an URL. -}
diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs
index 23b383a09d..1ecbb029b5 100644
--- a/Git/LsFiles.hs
+++ b/Git/LsFiles.hs
@@ -16,7 +16,7 @@ module Git.LsFiles (
 ) where
 
 import Git
-import Utility
+import Utility.SafeCommand
 
 {- Scans for files that are checked into git at the specified locations. -}
 inRepo :: Repo -> [FilePath] -> IO [FilePath]
diff --git a/Git/Queue.hs b/Git/Queue.hs
index 0016be4727..25b9ffad08 100644
--- a/Git/Queue.hs
+++ b/Git/Queue.hs
@@ -19,15 +19,15 @@ import System.IO
 import System.Cmd.Utils
 import Data.String.Utils
 import Control.Monad (forM_)
-import Utility
+import Utility.SafeCommand
 
 import Git
 
 {- An action to perform in a git repository. The file to act on
  - is not included, and must be able to be appended after the params. -}
-data Action = Action {
-		getSubcommand :: String,
-		getParams :: [CommandParam]
+data Action = Action
+	{ getSubcommand :: String
+	, getParams :: [CommandParam]
 	} deriving (Show, Eq, Ord)
 
 {- A queue of actions to perform (in any order) on a git repository,
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index b0da071703..a5bcbeac4a 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -18,7 +18,7 @@ import Data.Maybe
 import Data.String.Utils
 
 import Git
-import Utility
+import Utility.SafeCommand
 
 {- Performs a union merge between two branches, staging it in the index.
  - Any previously staged changes in the index will be lost.
diff --git a/Init.hs b/Init.hs
index a469657a11..2067c524cf 100644
--- a/Init.hs
+++ b/Init.hs
@@ -22,6 +22,7 @@ import Version
 import Messages
 import Types
 import Utility
+import Utility.Conditional
 import UUID
 
 initialize :: Annex ()
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 069209792d..ebb4b10a5b 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -29,6 +29,8 @@ import UUID
 import Locations
 import Config
 import Utility
+import Utility.Conditional
+import Utility.SafeCommand
 import Messages
 import Utility.Ssh
 import Remote.Helper.Special
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index fd227f85d8..7ddb60462f 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -27,6 +27,8 @@ import Utility.CopyFile
 import Config
 import Content
 import Utility
+import Utility.Conditional
+import Utility.Path
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 80ba8a153d..fadd48a036 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -26,6 +26,8 @@ import Messages
 import Utility.CopyFile
 import Utility.RsyncFile
 import Utility.Ssh
+import Utility.SafeCommand
+import Utility.Path
 import qualified Utility.Url as Url
 import Config
 import Init
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index c302a0ff5e..b842588c0c 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -17,7 +17,7 @@ import Types.Remote
 import qualified Git
 import qualified Annex
 import UUID
-import Utility
+import Utility.SafeCommand
 
 {- Special remotes don't have a configured url, so Git.Repo does not
  - automatically generate remotes for them. This looks for a different
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index ef52d0482b..54b9806ffc 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -28,6 +28,7 @@ import Locations
 import Config
 import Content
 import Utility
+import Utility.SafeCommand
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index eba67e3fd6..3707966ad6 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -26,11 +26,14 @@ import Locations
 import Config
 import Content
 import Utility
+import Utility.Conditional
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
 import Messages
 import Utility.RsyncFile
+import Utility.SafeCommand
+import Utility.Path
 
 type RsyncUrl = String
 
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 160d9309fe..b4567a0b71 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -31,6 +31,8 @@ import Backend
 import Messages
 import Version
 import Utility
+import Utility.SafeCommand
+import Utility.Path
 import qualified Upgrade.V2
 
 -- v2 adds hashing of filenames of content and location log files.
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 0b1d69f8e1..ffd0f06535 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -20,6 +20,8 @@ import qualified Git
 import qualified Branch
 import Messages
 import Utility
+import Utility.Conditional
+import Utility.SafeCommand
 import LocationLog
 import Content
 
diff --git a/Utility.hs b/Utility.hs
index 8a332601b7..788dc41030 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -6,20 +6,8 @@
  -}
 
 module Utility (
-	CommandParam(..),
-	toCommand,
 	hGetContentsStrict,
 	readFileStrict,
-	parentDir,
-	absPath,
-	absPathFrom,
-	relPathCwdToFile,
-	relPathDirToFile,
-	boolSystem,
-	boolSystemEnv,
-	executeFile,
-	shellEscape,
-	shellUnEscape,
 	unsetFileMode,
 	readMaybe,
 	viaTmp,
@@ -27,125 +15,19 @@ module Utility (
 	dirContains,
 	dirContents,
 	myHomeDir,
-	catchBool,
-	whenM,
-	(>>?),
-	unlessM,
-	(>>!),
-	
-	prop_idempotent_shellEscape,
-	prop_idempotent_shellEscape_multiword,
-	prop_parentDir_basics,
-	prop_relPathDirToFile_basics
+	catchBool
 ) where
 
 import IO (bracket)
 import System.IO
-import System.Exit
-import qualified System.Posix.Process
 import System.Posix.Process hiding (executeFile)
-import System.Posix.Signals
 import System.Posix.Files
 import System.Posix.Types
 import System.Posix.User
-import Data.String.Utils
-import System.Path
 import System.FilePath
 import System.Directory
 import Foreign (complement)
-import Data.List
-import Data.Maybe
-import Control.Monad (liftM2, when, unless)
-import System.Log.Logger
-
-{- A type for parameters passed to a shell command. A command can
- - be passed either some Params (multiple parameters can be included,
- - whitespace-separated, or a single Param (for when parameters contain
- - whitespace), or a File.
- -}
-data CommandParam = Params String | Param String | File FilePath
-	deriving (Eq, Show, Ord)
-
-{- Used to pass a list of CommandParams to a function that runs
- - a command and expects Strings. -}
-toCommand :: [CommandParam] -> [String]
-toCommand = (>>= unwrap)
-	where
-		unwrap (Param s) = [s]
-		unwrap (Params s) = filter (not . null) (split " " s)
-		-- Files that start with a dash are modified to avoid
-		-- the command interpreting them as options.
-		unwrap (File s@('-':_)) = ["./" ++ s]
-		unwrap (File s) = [s]
-
-{- Run a system command, and returns True or False
- - if it succeeded or failed.
- -
- - SIGINT(ctrl-c) is allowed to propigate and will terminate the program.
- -}
-boolSystem :: FilePath -> [CommandParam] -> IO Bool
-boolSystem command params = boolSystemEnv command params Nothing
-
-boolSystemEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO Bool
-boolSystemEnv command params env = do
-	-- Going low-level because all the high-level system functions
-	-- block SIGINT etc. We need to block SIGCHLD, but allow
-	-- SIGINT to do its default program termination.
-	let sigset = addSignal sigCHLD emptySignalSet
-	oldint <- installHandler sigINT Default Nothing
-	oldset <- getSignalMask
-	blockSignals sigset
-	childpid <- forkProcess $ childaction oldint oldset
-	mps <- getProcessStatus True False childpid
-	restoresignals oldint oldset
-	case mps of
-		Just (Exited ExitSuccess) -> return True
-		_ -> return False
-	where
-		restoresignals oldint oldset = do
-			_ <- installHandler sigINT oldint Nothing
-			setSignalMask oldset
-		childaction oldint oldset = do
-			restoresignals oldint oldset
-			executeFile command True (toCommand params) env
-
-{- executeFile with debug logging -}
-executeFile :: FilePath -> Bool -> [String] -> Maybe [(String, String)] -> IO ()
-executeFile c path p e = do
-	debugM "Utility.executeFile" $
-		"Running: " ++ c ++ " " ++ show p ++ " " ++ maybe "" show e
-	System.Posix.Process.executeFile c path p e
-
-{- Escapes a filename or other parameter to be safely able to be exposed to
- - the shell. -}
-shellEscape :: String -> String
-shellEscape f = "'" ++ escaped ++ "'"
-	where
-		-- replace ' with '"'"'
-		escaped = join "'\"'\"'" $ split "'" f
-
-{- Unescapes a set of shellEscaped words or filenames. -}
-shellUnEscape :: String -> [String]
-shellUnEscape [] = []
-shellUnEscape s = word : shellUnEscape rest
-	where
-		(word, rest) = findword "" s
-		findword w [] = (w, "")
-		findword w (c:cs)
-			| c == ' ' = (w, cs)
-			| c == '\'' = inquote c w cs
-			| c == '"' = inquote c w cs
-			| otherwise = findword (w++[c]) cs
-		inquote _ w [] = (w, "")
-		inquote q w (c:cs)
-			| c == q = findword w cs
-			| otherwise = inquote q (w++[c]) cs
-
-{- For quickcheck. -}
-prop_idempotent_shellEscape :: String -> Bool
-prop_idempotent_shellEscape s = [s] == (shellUnEscape . shellEscape) s
-prop_idempotent_shellEscape_multiword :: [String] -> Bool
-prop_idempotent_shellEscape_multiword s = s == (shellUnEscape . unwords . map shellEscape) s
+import Utility.Path
 
 {- A version of hgetContents that is not lazy. Ensures file is 
  - all read before it gets closed. -}
@@ -156,82 +38,6 @@ hGetContentsStrict h  = hGetContents h >>= \s -> length s `seq` return s
 readFileStrict :: FilePath -> IO String
 readFileStrict f = readFile f >>= \s -> length s `seq` return s
 
-{- Returns the parent directory of a path. Parent of / is "" -}
-parentDir :: FilePath -> FilePath
-parentDir dir =
-	if not $ null dirs
-	then slash ++ join s (take (length dirs - 1) dirs)
-	else ""
-		where
-			dirs = filter (not . null) $ split s dir
-			slash = if isAbsolute dir then s else ""
-			s = [pathSeparator]
-
-prop_parentDir_basics :: FilePath -> Bool
-prop_parentDir_basics dir
-	| null dir = True
-	| dir == "/" = parentDir dir == ""
-	| otherwise = p /= dir
-	where
-		p = parentDir dir
-
-{- Checks if the first FilePath is, or could be said to contain the second.
- - For example, "foo/" contains "foo/bar". Also, "foo", "./foo", "foo/" etc
- - are all equivilant.
- -}
-dirContains :: FilePath -> FilePath -> Bool
-dirContains a b = a == b || a' == b' || (a'++"/") `isPrefixOf` b'
-	where
-		norm p = fromMaybe "" $ absNormPath p "."
-		a' = norm a
-		b' = norm b
-
-{- Converts a filename into a normalized, absolute path. -}
-absPath :: FilePath -> IO FilePath
-absPath file = do
-	cwd <- getCurrentDirectory
-	return $ absPathFrom cwd file
-
-{- Converts a filename into a normalized, absolute path
- - from the specified cwd. -}
-absPathFrom :: FilePath -> FilePath -> FilePath
-absPathFrom cwd file = fromMaybe bad $ absNormPath cwd file
-	where
-		bad = error $ "unable to normalize " ++ file
-
-{- Constructs a relative path from the CWD to a file.
- -
- - For example, assuming CWD is /tmp/foo/bar:
- -    relPathCwdToFile "/tmp/foo" == ".."
- -    relPathCwdToFile "/tmp/foo/bar" == "" 
- -}
-relPathCwdToFile :: FilePath -> IO FilePath
-relPathCwdToFile f = liftM2 relPathDirToFile getCurrentDirectory (absPath f)
-
-{- Constructs a relative path from a directory to a file.
- -
- - Both must be absolute, and normalized (eg with absNormpath).
- -}
-relPathDirToFile :: FilePath -> FilePath -> FilePath
-relPathDirToFile from to = path
-	where
-		s = [pathSeparator]
-		pfrom = split s from
-		pto = split s to
-		common = map fst $ filter same $ zip pfrom pto
-		same (c,d) = c == d
-		uncommon = drop numcommon pto
-		dotdots = replicate (length pfrom - numcommon) ".."
-		numcommon = length common
-		path = join s $ dotdots ++ uncommon
-
-prop_relPathDirToFile_basics :: FilePath -> FilePath -> Bool
-prop_relPathDirToFile_basics from to
-	| from == to = null r
-	| otherwise = not (null r)
-	where
-		r = relPathDirToFile from to 
-
 {- Removes a FileMode from a file.
  - For example, call with otherWriteMode to chmod o-w -}
 unsetFileMode :: FilePath -> FileMode -> IO ()
@@ -288,20 +94,3 @@ myHomeDir = do
 {- Catches IO errors and returns a Bool -}
 catchBool :: IO Bool -> IO Bool
 catchBool = flip catch (const $ return False)
-
-{- when with a monadic conditional -}
-whenM :: Monad m => m Bool -> m () -> m ()
-whenM c a = c >>= flip when a
-
-unlessM :: Monad m => m Bool -> m () -> m ()
-unlessM c a = c >>= flip unless a
-
-(>>?) :: Monad m => m Bool -> m () -> m ()
-(>>?) = whenM
-
-(>>!) :: Monad m => m Bool -> m () -> m ()
-(>>!) = unlessM
-
--- low fixity allows eg, foo bar >>! error $ "failed " ++ meep
-infixr 0 >>?
-infixr 0 >>!
diff --git a/Utility/Conditional.hs b/Utility/Conditional.hs
new file mode 100644
index 0000000000..85e39ec64c
--- /dev/null
+++ b/Utility/Conditional.hs
@@ -0,0 +1,26 @@
+{- monadic conditional operators
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Conditional where
+
+import Control.Monad (when, unless)
+
+whenM :: Monad m => m Bool -> m () -> m ()
+whenM c a = c >>= flip when a
+
+unlessM :: Monad m => m Bool -> m () -> m ()
+unlessM c a = c >>= flip unless a
+
+(>>?) :: Monad m => m Bool -> m () -> m ()
+(>>?) = whenM
+
+(>>!) :: Monad m => m Bool -> m () -> m ()
+(>>!) = unlessM
+
+-- low fixity allows eg, foo bar >>! error $ "failed " ++ meep
+infixr 0 >>?
+infixr 0 >>!
diff --git a/Utility/CopyFile.hs b/Utility/CopyFile.hs
index befb00f8f2..9019357196 100644
--- a/Utility/CopyFile.hs
+++ b/Utility/CopyFile.hs
@@ -9,7 +9,8 @@ module Utility.CopyFile (copyFile) where
 
 import System.Directory (doesFileExist, removeFile)
 
-import Utility
+import Utility.Conditional
+import Utility.SafeCommand
 import qualified Build.SysConfig as SysConfig
 
 {- The cp command is used, because I hate reinventing the wheel,
diff --git a/Utility/Path.hs b/Utility/Path.hs
new file mode 100644
index 0000000000..517c175bc4
--- /dev/null
+++ b/Utility/Path.hs
@@ -0,0 +1,92 @@
+{- path manipulation
+ -
+ - Copyright 2010-2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Path where
+
+import Data.String.Utils
+import System.Path
+import System.FilePath
+import System.Directory
+import Data.List
+import Data.Maybe
+import Control.Monad (liftM2)
+
+{- Returns the parent directory of a path. Parent of / is "" -}
+parentDir :: FilePath -> FilePath
+parentDir dir =
+	if not $ null dirs
+	then slash ++ join s (take (length dirs - 1) dirs)
+	else ""
+		where
+			dirs = filter (not . null) $ split s dir
+			slash = if isAbsolute dir then s else ""
+			s = [pathSeparator]
+
+prop_parentDir_basics :: FilePath -> Bool
+prop_parentDir_basics dir
+	| null dir = True
+	| dir == "/" = parentDir dir == ""
+	| otherwise = p /= dir
+	where
+		p = parentDir dir
+
+{- Checks if the first FilePath is, or could be said to contain the second.
+ - For example, "foo/" contains "foo/bar". Also, "foo", "./foo", "foo/" etc
+ - are all equivilant.
+ -}
+dirContains :: FilePath -> FilePath -> Bool
+dirContains a b = a == b || a' == b' || (a'++"/") `isPrefixOf` b'
+	where
+		norm p = fromMaybe "" $ absNormPath p "."
+		a' = norm a
+		b' = norm b
+
+{- Converts a filename into a normalized, absolute path. -}
+absPath :: FilePath -> IO FilePath
+absPath file = do
+	cwd <- getCurrentDirectory
+	return $ absPathFrom cwd file
+
+{- Converts a filename into a normalized, absolute path
+ - from the specified cwd. -}
+absPathFrom :: FilePath -> FilePath -> FilePath
+absPathFrom cwd file = fromMaybe bad $ absNormPath cwd file
+	where
+		bad = error $ "unable to normalize " ++ file
+
+{- Constructs a relative path from the CWD to a file.
+ -
+ - For example, assuming CWD is /tmp/foo/bar:
+ -    relPathCwdToFile "/tmp/foo" == ".."
+ -    relPathCwdToFile "/tmp/foo/bar" == "" 
+ -}
+relPathCwdToFile :: FilePath -> IO FilePath
+relPathCwdToFile f = liftM2 relPathDirToFile getCurrentDirectory (absPath f)
+
+{- Constructs a relative path from a directory to a file.
+ -
+ - Both must be absolute, and normalized (eg with absNormpath).
+ -}
+relPathDirToFile :: FilePath -> FilePath -> FilePath
+relPathDirToFile from to = path
+	where
+		s = [pathSeparator]
+		pfrom = split s from
+		pto = split s to
+		common = map fst $ filter same $ zip pfrom pto
+		same (c,d) = c == d
+		uncommon = drop numcommon pto
+		dotdots = replicate (length pfrom - numcommon) ".."
+		numcommon = length common
+		path = join s $ dotdots ++ uncommon
+
+prop_relPathDirToFile_basics :: FilePath -> FilePath -> Bool
+prop_relPathDirToFile_basics from to
+	| from == to = null r
+	| otherwise = not (null r)
+	where
+		r = relPathDirToFile from to 
diff --git a/Utility/RsyncFile.hs b/Utility/RsyncFile.hs
index 6e21ba0632..b6c2267e87 100644
--- a/Utility/RsyncFile.hs
+++ b/Utility/RsyncFile.hs
@@ -9,7 +9,7 @@ module Utility.RsyncFile where
 
 import Data.String.Utils
 
-import Utility
+import Utility.SafeCommand
 
 {- Generates parameters to make rsync use a specified command as its remote
  - shell. -}
diff --git a/Utility/SafeCommand.hs b/Utility/SafeCommand.hs
new file mode 100644
index 0000000000..ba9362603e
--- /dev/null
+++ b/Utility/SafeCommand.hs
@@ -0,0 +1,104 @@
+{- safely running shell commands
+ -
+ - Copyright 2010-2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.SafeCommand where
+
+import System.Exit
+import qualified System.Posix.Process
+import System.Posix.Process hiding (executeFile)
+import System.Posix.Signals
+import Data.String.Utils
+import System.Log.Logger
+
+{- A type for parameters passed to a shell command. A command can
+ - be passed either some Params (multiple parameters can be included,
+ - whitespace-separated, or a single Param (for when parameters contain
+ - whitespace), or a File.
+ -}
+data CommandParam = Params String | Param String | File FilePath
+	deriving (Eq, Show, Ord)
+
+{- Used to pass a list of CommandParams to a function that runs
+ - a command and expects Strings. -}
+toCommand :: [CommandParam] -> [String]
+toCommand = (>>= unwrap)
+	where
+		unwrap (Param s) = [s]
+		unwrap (Params s) = filter (not . null) (split " " s)
+		-- Files that start with a dash are modified to avoid
+		-- the command interpreting them as options.
+		unwrap (File s@('-':_)) = ["./" ++ s]
+		unwrap (File s) = [s]
+
+{- Run a system command, and returns True or False
+ - if it succeeded or failed.
+ -
+ - SIGINT(ctrl-c) is allowed to propigate and will terminate the program.
+ -}
+boolSystem :: FilePath -> [CommandParam] -> IO Bool
+boolSystem command params = boolSystemEnv command params Nothing
+
+boolSystemEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO Bool
+boolSystemEnv command params env = do
+	-- Going low-level because all the high-level system functions
+	-- block SIGINT etc. We need to block SIGCHLD, but allow
+	-- SIGINT to do its default program termination.
+	let sigset = addSignal sigCHLD emptySignalSet
+	oldint <- installHandler sigINT Default Nothing
+	oldset <- getSignalMask
+	blockSignals sigset
+	childpid <- forkProcess $ childaction oldint oldset
+	mps <- getProcessStatus True False childpid
+	restoresignals oldint oldset
+	case mps of
+		Just (Exited ExitSuccess) -> return True
+		_ -> return False
+	where
+		restoresignals oldint oldset = do
+			_ <- installHandler sigINT oldint Nothing
+			setSignalMask oldset
+		childaction oldint oldset = do
+			restoresignals oldint oldset
+			executeFile command True (toCommand params) env
+
+{- executeFile with debug logging -}
+executeFile :: FilePath -> Bool -> [String] -> Maybe [(String, String)] -> IO ()
+executeFile c path p e = do
+	debugM "Utility.SafeCommand.executeFile" $
+		"Running: " ++ c ++ " " ++ show p ++ " " ++ maybe "" show e
+	System.Posix.Process.executeFile c path p e
+
+{- Escapes a filename or other parameter to be safely able to be exposed to
+ - the shell. -}
+shellEscape :: String -> String
+shellEscape f = "'" ++ escaped ++ "'"
+	where
+		-- replace ' with '"'"'
+		escaped = join "'\"'\"'" $ split "'" f
+
+{- Unescapes a set of shellEscaped words or filenames. -}
+shellUnEscape :: String -> [String]
+shellUnEscape [] = []
+shellUnEscape s = word : shellUnEscape rest
+	where
+		(word, rest) = findword "" s
+		findword w [] = (w, "")
+		findword w (c:cs)
+			| c == ' ' = (w, cs)
+			| c == '\'' = inquote c w cs
+			| c == '"' = inquote c w cs
+			| otherwise = findword (w++[c]) cs
+		inquote _ w [] = (w, "")
+		inquote q w (c:cs)
+			| c == q = findword w cs
+			| otherwise = inquote q (w++[c]) cs
+
+{- For quickcheck. -}
+prop_idempotent_shellEscape :: String -> Bool
+prop_idempotent_shellEscape s = [s] == (shellUnEscape . shellEscape) s
+prop_idempotent_shellEscape_multiword :: [String] -> Bool
+prop_idempotent_shellEscape_multiword s = s == (shellUnEscape . unwords . map shellEscape) s
diff --git a/Utility/Ssh.hs b/Utility/Ssh.hs
index 6cbc362a03..05269552c7 100644
--- a/Utility/Ssh.hs
+++ b/Utility/Ssh.hs
@@ -10,7 +10,7 @@ module Utility.Ssh where
 import Control.Monad.State (liftIO)
 
 import qualified Git
-import Utility
+import Utility.SafeCommand
 import Types
 import Config
 
diff --git a/Utility/Url.hs b/Utility/Url.hs
index 5954e0ff7e..69b53c34c0 100644
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -19,7 +19,7 @@ import Network.URI
 
 import Types
 import Messages
-import Utility
+import Utility.SafeCommand
 
 type URLString = String
 
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 29ac63aea1..1fb928f9da 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -11,7 +11,8 @@ import Data.List
 import qualified Git
 import CmdLine
 import Command
-import Utility
+import Utility.Conditional
+import Utility.SafeCommand
 import Options
 
 import qualified Command.ConfigList
diff --git a/test.hs b/test.hs
index 2352df36a6..e4e1fb131d 100644
--- a/test.hs
+++ b/test.hs
@@ -24,11 +24,12 @@ import qualified Data.Map as M
 import System.Path (recurseDir)
 import System.IO.HVFS (SystemFS(..))
 
+import Utility.SafeCommand
+
 import qualified Annex
 import qualified Backend
 import qualified Git
 import qualified Locations
-import qualified Utility
 import qualified Types.Backend
 import qualified Types
 import qualified GitAnnex
@@ -42,6 +43,7 @@ import qualified Command.DropUnused
 import qualified Types.Key
 import qualified Config
 import qualified Crypto
+import qualified Utility.Path
 
 -- for quickcheck
 instance Arbitrary Types.Key.Key where
@@ -72,11 +74,12 @@ quickcheck = TestLabel "quickcheck" $ TestList
 	[ qctest "prop_idempotent_deencode" Git.prop_idempotent_deencode
 	, qctest "prop_idempotent_fileKey" Locations.prop_idempotent_fileKey
 	, qctest "prop_idempotent_key_read_show" Types.Key.prop_idempotent_key_read_show
-	, qctest "prop_idempotent_shellEscape" Utility.prop_idempotent_shellEscape
-	, qctest "prop_idempotent_shellEscape_multiword" Utility.prop_idempotent_shellEscape_multiword
+	, qctest "prop_idempotent_shellEscape" Utility.SafeCommand.prop_idempotent_shellEscape
+	, qctest "prop_idempotent_shellEscape_multiword" Utility.SafeCommand.prop_idempotent_shellEscape_multiword
 	, qctest "prop_idempotent_configEscape" RemoteLog.prop_idempotent_configEscape
-	, qctest "prop_parentDir_basics" Utility.prop_parentDir_basics
-	, qctest "prop_relPathDirToFile_basics" Utility.prop_relPathDirToFile_basics
+	, qctest "prop_parentDir_basics" Utility.Path.prop_parentDir_basics
+
+	, qctest "prop_relPathDirToFile_basics" Utility.Path.prop_relPathDirToFile_basics
 	, qctest "prop_cost_sane" Config.prop_cost_sane
 	, qctest "prop_hmacWithCipher_sane" Crypto.prop_hmacWithCipher_sane
 	]
@@ -117,8 +120,8 @@ test_add = "git-annex add" ~: TestList [basic, sha1dup, subdirs]
 			git_annex "add" ["-q", annexedfile] @? "add failed"
 			annexed_present annexedfile
 			writeFile ingitfile $ content ingitfile
-			Utility.boolSystem "git" [Utility.Param "add", Utility.File ingitfile] @? "git add failed"
-			Utility.boolSystem "git" [Utility.Params "commit -q -a -m commit"] @? "git commit failed"
+			boolSystem "git" [Param "add", File ingitfile] @? "git add failed"
+			boolSystem "git" [Params "commit -q -a -m commit"] @? "git commit failed"
 			git_annex "add" ["-q", ingitfile] @? "add ingitfile should be no-op"
 			unannexed ingitfile
 		sha1dup = TestCase $ intmpclonerepo $ do
@@ -145,7 +148,7 @@ test_setkey = "git-annex setkey/fromkey" ~: TestCase $ inmainrepo $ do
 	let key = show $ fromJust r
 	git_annex "setkey" ["-q", "--key", key, tmp] @? "setkey failed"
 	git_annex "fromkey" ["-q", "--key", key, sha1annexedfile] @? "fromkey failed"
-	Utility.boolSystem "git" [Utility.Params "commit -q -a -m commit"] @? "git commit failed"
+	boolSystem "git" [Params "commit -q -a -m commit"] @? "git commit failed"
 	annexed_present sha1annexedfile
 	where
 		tmp = "tmpfile"
@@ -172,7 +175,7 @@ test_drop = "git-annex drop" ~: TestList [noremote, withremote, untrustedremote]
 	where
 		noremote = "no remotes" ~: TestCase $ intmpclonerepo $ do
 			git_annex "get" ["-q", annexedfile] @? "get failed"
-			Utility.boolSystem "git" [Utility.Params "remote rm origin"]
+			boolSystem "git" [Params "remote rm origin"]
 				@? "git remote rm origin failed"
 			r <- git_annex "drop" ["-q", annexedfile]
 			not r @? "drop wrongly succeeded with no known copy of file"
@@ -303,12 +306,12 @@ test_edit = "git-annex edit/commit" ~: TestList [t False, t True]
 			then do
 				-- pre-commit depends on the file being
 				-- staged, normally git commit does this
-				Utility.boolSystem "git" [Utility.Param "add", Utility.File annexedfile]
+				boolSystem "git" [Param "add", File annexedfile]
 					@? "git add of edited file failed"
 				git_annex "pre-commit" ["-q"]
 					@? "pre-commit failed"
 			else do
-				Utility.boolSystem "git" [Utility.Params "commit -q -a -m contentchanged"]
+				boolSystem "git" [Params "commit -q -a -m contentchanged"]
 					@? "git commit of edited file failed"
 		runchecks [checklink, checkunwritable] annexedfile
 		c <- readFile annexedfile
@@ -326,7 +329,7 @@ test_fix = "git-annex fix" ~: intmpclonerepo $ do
 	git_annex "fix" ["-q", annexedfile] @? "fix of present file failed"
 	annexed_present annexedfile
 	createDirectory subdir
-	Utility.boolSystem "git" [Utility.Param "mv", Utility.File annexedfile, Utility.File subdir]
+	boolSystem "git" [Param "mv", File annexedfile, File subdir]
 		@? "git mv failed"
 	git_annex "fix" ["-q", newfile] @? "fix of moved file failed"
 	runchecks [checklink, checkunwritable] newfile
@@ -364,9 +367,9 @@ test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withrem
 	where
 		basicfsck = TestCase $ intmpclonerepo $ do
 			git_annex "fsck" ["-q"] @? "fsck failed"
-			Utility.boolSystem "git" [Utility.Params "config annex.numcopies 2"] @? "git config failed"
+			boolSystem "git" [Params "config annex.numcopies 2"] @? "git config failed"
 			fsck_should_fail "numcopies unsatisfied"
-			Utility.boolSystem "git" [Utility.Params "config annex.numcopies 1"] @? "git config failed"
+			boolSystem "git" [Params "config annex.numcopies 1"] @? "git config failed"
 			corrupt annexedfile
 			corrupt sha1annexedfile
 		withlocaluntrusted = TestCase $ intmpclonerepo $ do
@@ -377,7 +380,7 @@ test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withrem
 			git_annex "trust" ["-q", "."] @? "trust of current repo failed"
 			git_annex "fsck" ["-q", annexedfile] @? "fsck failed on file present in trusted repo"
 		withremoteuntrusted = TestCase $ intmpclonerepo $ do
-			Utility.boolSystem "git" [Utility.Params "config annex.numcopies 2"] @? "git config failed"
+			boolSystem "git" [Params "config annex.numcopies 2"] @? "git config failed"
 			git_annex "get" ["-q", annexedfile] @? "get failed"
 			git_annex "get" ["-q", sha1annexedfile] @? "get failed"
 			git_annex "fsck" ["-q"] @? "fsck failed with numcopies=2 and 2 copies"
@@ -448,9 +451,9 @@ test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do
 	git_annex "get" ["-q", annexedfile] @? "get of file failed"
 	git_annex "get" ["-q", sha1annexedfile] @? "get of file failed"
 	checkunused []
-	Utility.boolSystem "git" [Utility.Params "rm -q", Utility.File annexedfile] @? "git rm failed"
+	boolSystem "git" [Params "rm -q", File annexedfile] @? "git rm failed"
 	checkunused [annexedfilekey]
-	Utility.boolSystem "git" [Utility.Params "rm -q", Utility.File sha1annexedfile] @? "git rm failed"
+	boolSystem "git" [Params "rm -q", File sha1annexedfile] @? "git rm failed"
 	checkunused [annexedfilekey, sha1annexedfilekey]
 
 	-- good opportunity to test dropkey also
@@ -526,10 +529,10 @@ setuprepo :: FilePath -> IO FilePath
 setuprepo dir = do
 	cleanup dir
 	ensuretmpdir
-	Utility.boolSystem "git" [Utility.Params "init -q", Utility.File dir] @? "git init failed"
+	boolSystem "git" [Params "init -q", File dir] @? "git init failed"
 	indir dir $ do
-		Utility.boolSystem "git" [Utility.Params "config user.name", Utility.Param "Test User"] @? "git config failed"
-		Utility.boolSystem "git" [Utility.Params "config user.email test@example.com"] @? "git config failed"
+		boolSystem "git" [Params "config user.name", Param "Test User"] @? "git config failed"
+		boolSystem "git" [Params "config user.email test@example.com"] @? "git config failed"
 	return dir
 
 -- clones are always done as local clones; we cannot test ssh clones
@@ -537,7 +540,7 @@ clonerepo :: FilePath -> FilePath -> IO FilePath
 clonerepo old new = do
 	cleanup new
 	ensuretmpdir
-	Utility.boolSystem "git" [Utility.Params "clone -q", Utility.File old, Utility.File new] @? "git clone failed"
+	boolSystem "git" [Params "clone -q", File old, File new] @? "git clone failed"
 	indir new $ git_annex "init" ["-q", new] @? "git annex init failed"
 	return new
 	

From 20259c2955e408a72e0960207fc8be4cbeec2e21 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 23 Aug 2011 13:41:32 -0400
Subject: [PATCH 069/111] Set EMAIL when running test suite so that git does
 not need to be configured first. Closes: #638998

---
 debian/changelog | 7 +++++++
 test.hs          | 3 +++
 2 files changed, 10 insertions(+)

diff --git a/debian/changelog b/debian/changelog
index a8280a156d..bb06e1fba9 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+git-annex (3.20110820) UNRELEASED; urgency=low
+
+  * Set EMAIL when running test suite so that git does not need to be
+    configured first. Closes: #638998
+
+ -- Joey Hess   Tue, 23 Aug 2011 13:41:01 -0400
+
 git-annex (3.20110819) unstable; urgency=low
 
   * Now "git annex init" only has to be run once, when a git repository
diff --git a/test.hs b/test.hs
index e4e1fb131d..4d751a707b 100644
--- a/test.hs
+++ b/test.hs
@@ -646,6 +646,9 @@ prepare = do
 	p <- getEnvDefault  "PATH" ""
 	setEnv "PATH" (cwd ++ ":" ++ p) True
 	setEnv "TOPDIR" cwd True
+	-- Avoid git complaining if it cannot determine the user's email
+	-- address.
+	setEnv "EMAIL" "git-annex test " True
 
 changeToTmpDir :: FilePath -> IO ()
 changeToTmpDir t = do

From 678726c10c13481c082743808a5188d28567e2b3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 25 Aug 2011 00:28:55 -0400
Subject: [PATCH 070/111] code simplification thanks to applicative functors

---
 Annex.hs           |  7 +++++--
 Branch.hs          |  7 ++++---
 Command.hs         |  7 ++++---
 Command/Migrate.hs |  3 ++-
 Command/Status.hs  | 10 ++++------
 Config.hs          |  6 +++---
 Crypto.hs          |  3 ++-
 Git.hs             |  3 ++-
 LocationLog.hs     |  3 ++-
 PresenceLog.hs     |  3 ++-
 Remote.hs          | 13 +++++++------
 RemoteLog.hs       |  3 ++-
 Upgrade/V1.hs      |  5 +++--
 Utility/Path.hs    |  4 ++--
 Utility/Url.hs     |  5 ++---
 15 files changed, 46 insertions(+), 36 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 07316bd370..287aed875e 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -20,6 +20,7 @@ module Annex (
 
 import Control.Monad.State
 import Control.Monad.IO.Control
+import Control.Applicative hiding (empty)
 
 import qualified Git
 import Git.Queue
@@ -36,7 +37,9 @@ newtype Annex a = Annex { runAnnex :: StateT AnnexState IO a }
 		Monad,
 		MonadIO,
 		MonadControlIO,
-		MonadState AnnexState
+		MonadState AnnexState,
+		Functor,
+		Applicative
 	)
 
 -- internal state storage
@@ -83,7 +86,7 @@ newState gitrepo = AnnexState
 
 {- Create and returns an Annex state object for the specified git repo. -}
 new :: Git.Repo -> IO AnnexState
-new gitrepo = newState `liftM` Git.configRead gitrepo
+new gitrepo = newState <$> Git.configRead gitrepo
 
 {- performs an action in the Annex monad -}
 run :: AnnexState -> Annex a -> IO (a, AnnexState)
diff --git a/Branch.hs b/Branch.hs
index d5bfe1b09f..5008b2e200 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -20,6 +20,7 @@ module Branch (
 
 import Control.Monad (when, unless, liftM)
 import Control.Monad.State (liftIO)
+import Control.Applicative ((<$>))
 import System.FilePath
 import System.Directory
 import Data.String.Utils
@@ -158,7 +159,7 @@ update = do
 		staged <- stageJournalFiles
 
 		refs <- siblingBranches
-		updated <- catMaybes `liftM` mapM updateRef refs
+		updated <- catMaybes <$> mapM updateRef refs
 		g <- Annex.gitRepo
 		unless (null updated && not staged) $ liftIO $
 			Git.commit g "update" fullname (fullname:updated)
@@ -182,7 +183,7 @@ hasOrigin = refExists originname
 
 {- Does the git-annex branch or a foo/git-annex branch exist? -}
 hasSomeBranch :: Annex Bool
-hasSomeBranch = liftM (not . null) siblingBranches
+hasSomeBranch = not . null <$> siblingBranches
 
 {- List of all git-annex branches, including the main one and any
  - from remotes. -}
@@ -323,7 +324,7 @@ getJournalFile file = do
 
 {- List of journal files. -}
 getJournalFiles :: Annex [FilePath]
-getJournalFiles = liftM (map fileJournal) getJournalFilesRaw
+getJournalFiles = map fileJournal <$> getJournalFilesRaw
 
 getJournalFilesRaw :: Annex [FilePath]
 getJournalFilesRaw = do
diff --git a/Command.hs b/Command.hs
index d3c1640ee0..21c50f9c02 100644
--- a/Command.hs
+++ b/Command.hs
@@ -11,6 +11,7 @@ import Control.Monad.State (liftIO)
 import System.Directory
 import System.Posix.Files
 import Control.Monad (filterM, liftM, when)
+import Control.Applicative
 import System.Path.WildMatch
 import Text.Regex.PCRE.Light.Char8
 import Data.List
@@ -183,7 +184,7 @@ withNothing a [] = return [a]
 withNothing _ _ = error "This command takes no parameters."
 
 backendPairs :: CommandSeekBackendFiles
-backendPairs a files = liftM (map a) $ Backend.chooseBackends files
+backendPairs a files = map a <$> Backend.chooseBackends files
 
 {- Filter out files those matching the exclude glob pattern,
  - if it was specified. -}
@@ -204,7 +205,7 @@ wildsRegex ws = compile regex []
 
 {- filter out symlinks -}	
 notSymlink :: FilePath -> IO Bool
-notSymlink f = liftM (not . isSymbolicLink) $ liftIO $ getSymbolicLinkStatus f
+notSymlink f = liftIO $ not . isSymbolicLink <$> getSymbolicLinkStatus f
 
 {- Descriptions of params used in usage messages. -}
 paramRepeating :: String -> String
@@ -271,4 +272,4 @@ preserveOrder orig new = collect orig new
  - of git file list commands, that assumption tends to hold.
  -}
 runPreserveOrder :: ([FilePath] -> IO [FilePath]) -> [FilePath] -> IO [FilePath]
-runPreserveOrder a files = liftM (preserveOrder files) (a files)
+runPreserveOrder a files = preserveOrder files <$> a files
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 25227ae162..6ad7e239c9 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -8,6 +8,7 @@
 module Command.Migrate where
 
 import Control.Monad.State (liftIO)
+import Control.Applicative
 import System.Posix.Files
 import System.Directory
 import System.FilePath
@@ -39,7 +40,7 @@ start (file, b) = isAnnexed file $ \(key, oldbackend) -> do
 			next $ perform file key newbackend
 		else stop
 	where
-		choosebackend Nothing = return . head =<< Backend.orderedList
+		choosebackend Nothing = head <$> Backend.orderedList
 		choosebackend (Just backend) = return backend
 
 {- Checks if a key is upgradable to a newer representation. -}
diff --git a/Command/Status.hs b/Command/Status.hs
index aef4df2329..5c82744b10 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -8,6 +8,7 @@
 module Command.Status where
 
 import Control.Monad.State
+import Control.Applicative
 import Data.Maybe
 import System.IO
 import Data.List
@@ -112,12 +113,10 @@ total_annex_size = stat "total annex size" $
 	cachedKeysReferenced >>= keySizeSum
 
 local_annex_keys :: Stat
-local_annex_keys = stat "local annex keys" $ 
-	return . show . snd =<< cachedKeysPresent
+local_annex_keys = stat "local annex keys" $ show . snd <$> cachedKeysPresent
 
 total_annex_keys :: Stat
-total_annex_keys = stat "total annex keys" $
-	return . show . snd =<< cachedKeysReferenced
+total_annex_keys = stat "total annex keys" $ show . snd <$> cachedKeysReferenced
 
 tmp_size :: Stat
 tmp_size = staleSize "temporary directory size" gitAnnexTmpDir
@@ -126,8 +125,7 @@ bad_data_size :: Stat
 bad_data_size = staleSize "bad keys size" gitAnnexBadDir
 
 backend_usage :: Stat
-backend_usage = stat "backend usage" $
-	return . usage =<< cachedKeysReferenced
+backend_usage = stat "backend usage" $ usage <$> cachedKeysReferenced
 	where
 		usage (ks, _) = pp "" $ sort $ map swap $ splits ks
 		splits :: [Key] -> [(String, Integer)]
diff --git a/Config.hs b/Config.hs
index 12f6480470..b4f4c0b922 100644
--- a/Config.hs
+++ b/Config.hs
@@ -9,7 +9,7 @@ module Config where
 
 import Data.Maybe
 import Control.Monad.State (liftIO)
-import Control.Monad (liftM)
+import Control.Applicative
 import System.Cmd.Utils
 
 import qualified Git
@@ -47,8 +47,8 @@ remoteConfig r key = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex
 remoteCost :: Git.Repo -> Int -> Annex Int
 remoteCost r def = do
 	cmd <- getConfig r "cost-command" ""
-	return . safeparse =<< if not $ null cmd
-			then liftM snd $ liftIO $ pipeFrom "sh" ["-c", cmd]
+	safeparse <$> if not $ null cmd
+			then liftIO $ snd <$> pipeFrom "sh" ["-c", cmd]
 			else getConfig r "cost" ""
 	where
 		safeparse v
diff --git a/Crypto.hs b/Crypto.hs
index ed29747aa6..d789b44556 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -38,6 +38,7 @@ import System.IO
 import System.Posix.IO
 import System.Posix.Types
 import System.Posix.Process
+import Control.Applicative
 import Control.Concurrent
 import Control.Exception (finally)
 import System.Exit
@@ -136,7 +137,7 @@ encryptCipher (Cipher c) (KeyIds ks) = do
 {- Decrypting an EncryptedCipher is expensive; the Cipher should be cached. -}
 decryptCipher :: RemoteConfig -> EncryptedCipher -> IO Cipher
 decryptCipher _ (EncryptedCipher encipher _) = 
-	return . Cipher =<< gpgPipeStrict decrypt encipher
+	Cipher <$> gpgPipeStrict decrypt encipher
 	where
 		decrypt = [ Param "--decrypt" ]
 
diff --git a/Git.hs b/Git.hs
index 7155b26343..ab43504e1a 100644
--- a/Git.hs
+++ b/Git.hs
@@ -63,6 +63,7 @@ module Git (
 ) where
 
 import Control.Monad (unless, when)
+import Control.Applicative
 import System.Directory
 import System.FilePath
 import System.Posix.Directory
@@ -446,7 +447,7 @@ commit g message newref parentrefs = do
 		pipeWriteRead g (map Param $ ["commit-tree", tree] ++ ps) message
 	run g "update-ref" [Param newref, Param sha]
 	where
-		ignorehandle a = return . snd =<< a
+		ignorehandle a = snd <$> a
 		ps = concatMap (\r -> ["-p", r]) parentrefs
 
 {- Reads null terminated output of a git command (as enabled by the -z 
diff --git a/LocationLog.hs b/LocationLog.hs
index 768483fa1b..fa660c8b67 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -24,6 +24,7 @@ module LocationLog (
 
 import System.FilePath
 import Control.Monad (when)
+import Control.Applicative
 import Data.Maybe
 
 import qualified Git
@@ -49,7 +50,7 @@ keyLocations key = currentLog $ logFile key
 {- Finds all keys that have location log information.
  - (There may be duplicate keys in the list.) -}
 loggedKeys :: Annex [Key]
-loggedKeys = return . mapMaybe (logFileKey . takeFileName) =<< Branch.files
+loggedKeys = mapMaybe (logFileKey . takeFileName) <$> Branch.files
 
 {- The filename of the log file for a given key. -}
 logFile :: Key -> String
diff --git a/PresenceLog.hs b/PresenceLog.hs
index ccb75ff5be..e0c8729979 100644
--- a/PresenceLog.hs
+++ b/PresenceLog.hs
@@ -28,6 +28,7 @@ import Data.Time
 import System.Locale
 import qualified Data.Map as Map
 import Control.Monad.State (liftIO)
+import Control.Applicative
 
 import qualified Branch
 import Types
@@ -81,7 +82,7 @@ addLog file line = do
 {- Reads a log file.
  - Note that the LogLines returned may be in any order. -}
 readLog :: FilePath -> Annex [LogLine]
-readLog file = return . parseLog =<< Branch.get file
+readLog file = parseLog <$> Branch.get file
 
 parseLog :: String -> [LogLine]
 parseLog s = filter parsable $ map read $ lines s
diff --git a/Remote.hs b/Remote.hs
index 1a5006f6fb..2c883f1a8f 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -29,11 +29,12 @@ module Remote (
 	forceTrust
 ) where
 
-import Control.Monad (filterM, liftM2)
+import Control.Monad (filterM)
 import Data.List
 import qualified Data.Map as M
 import Data.String.Utils
 import Data.Maybe
+import Control.Applicative
 
 import Types
 import Types.Remote
@@ -111,10 +112,10 @@ nameToUUID "." = getUUID =<< Annex.gitRepo -- special case for current repo
 nameToUUID n = do
 	res <- byName' n
 	case res of
-		Left e -> return . fromMaybe (error e) =<< byDescription
+		Left e -> fromMaybe (error e) <$> byDescription
 		Right r -> return $ uuid r
 	where
-		byDescription = return . M.lookup n . invertMap =<< uuidMap
+		byDescription = M.lookup n . invertMap <$> uuidMap
 		invertMap = M.fromList . map swap . M.toList
 		swap (a, b) = (b, a)
 
@@ -124,10 +125,10 @@ prettyPrintUUIDs uuids = do
 	here <- getUUID =<< Annex.gitRepo
 	-- Show descriptions from the uuid log, falling back to remote names,
 	-- as some remotes may not be in the uuid log
-	m <- liftM2 M.union uuidMap $
-		return . M.fromList . map (\r -> (uuid r, name r)) =<< genList
+	m <- M.union <$> uuidMap <*> availMap
 	return $ unwords $ map (\u -> "\t" ++ prettify m u here ++ "\n") uuids
 	where
+		availMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList
 		prettify m u here = base ++ ishere
 			where
 				base = if not $ null $ findlog m u
@@ -147,7 +148,7 @@ remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs
 {- Cost ordered lists of remotes that the LocationLog indicate may have a key.
  -}
 keyPossibilities :: Key -> Annex [Remote Annex]
-keyPossibilities key = return . fst =<< keyPossibilities' False key
+keyPossibilities key = fst <$> keyPossibilities' False key
 
 {- Cost ordered lists of remotes that the LocationLog indicate may have a key.
  -
diff --git a/RemoteLog.hs b/RemoteLog.hs
index 69a82f4987..620c0d8757 100644
--- a/RemoteLog.hs
+++ b/RemoteLog.hs
@@ -19,6 +19,7 @@ import Data.List
 import qualified Data.Map as M
 import Data.Maybe
 import Data.Char
+import Control.Applicative
 
 import qualified Branch
 import Types
@@ -40,7 +41,7 @@ configSet u c = do
 
 {- Map of remotes by uuid containing key/value config maps. -}
 readRemoteLog :: Annex (M.Map UUID RemoteConfig)
-readRemoteLog = return . remoteLogParse =<< Branch.get remoteLog
+readRemoteLog = remoteLogParse <$> Branch.get remoteLog
 
 remoteLogParse :: String -> M.Map UUID RemoteConfig
 remoteLogParse s =
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index b4567a0b71..9c3fd99595 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -11,6 +11,7 @@ import System.IO.Error (try)
 import System.Directory
 import Control.Monad.State (liftIO)
 import Control.Monad (filterM, forM_, unless)
+import Control.Applicative
 import System.Posix.Files
 import System.FilePath
 import Data.String.Utils
@@ -192,7 +193,7 @@ writeLog1 :: FilePath -> [LogLine] -> IO ()
 writeLog1 file ls = viaTmp writeFile file (unlines $ map show ls)
 
 readLog1 :: FilePath -> IO [LogLine]
-readLog1 file = catch (return . parseLog =<< readFileStrict file) (const $ return [])
+readLog1 file = catch (parseLog <$> readFileStrict file) (const $ return [])
 
 lookupFile1 :: FilePath -> Annex (Maybe (Key, Backend Annex))
 lookupFile1 file = do
@@ -201,7 +202,7 @@ lookupFile1 file = do
 		Left _ -> return Nothing
 		Right l -> makekey l
 	where
-		getsymlink = return . takeFileName =<< readSymbolicLink file
+		getsymlink = takeFileName <$> readSymbolicLink file
 		makekey l = case maybeLookupBackendName bname of
 			Nothing -> do
 				unless (null kname || null bname ||
diff --git a/Utility/Path.hs b/Utility/Path.hs
index 517c175bc4..9b8041dad0 100644
--- a/Utility/Path.hs
+++ b/Utility/Path.hs
@@ -13,7 +13,7 @@ import System.FilePath
 import System.Directory
 import Data.List
 import Data.Maybe
-import Control.Monad (liftM2)
+import Control.Applicative
 
 {- Returns the parent directory of a path. Parent of / is "" -}
 parentDir :: FilePath -> FilePath
@@ -65,7 +65,7 @@ absPathFrom cwd file = fromMaybe bad $ absNormPath cwd file
  -    relPathCwdToFile "/tmp/foo/bar" == "" 
  -}
 relPathCwdToFile :: FilePath -> IO FilePath
-relPathCwdToFile f = liftM2 relPathDirToFile getCurrentDirectory (absPath f)
+relPathCwdToFile f = relPathDirToFile <$> getCurrentDirectory <*> absPath f
 
 {- Constructs a relative path from a directory to a file.
  -
diff --git a/Utility/Url.hs b/Utility/Url.hs
index 69b53c34c0..f678720ed8 100644
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -11,7 +11,7 @@ module Utility.Url (
 	get
 ) where
 
-import Control.Monad (liftM)
+import Control.Applicative
 import Control.Monad.State (liftIO)
 import qualified Network.Browser as Browser
 import Network.HTTP
@@ -64,7 +64,6 @@ request url requesttype = Browser.browse $ do
 	Browser.setErrHandler ignore
 	Browser.setOutHandler ignore
 	Browser.setAllowRedirects True
-	liftM snd $ Browser.request
-		(mkRequest requesttype url :: Request_String)
+	snd <$> Browser.request (mkRequest requesttype url :: Request_String)
 	where
 		ignore = const $ return ()

From 9170e1b87d1eac000e9ea4d3944a73209ba9c718 Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Fri, 26 Aug 2011 10:55:43 +0000
Subject: [PATCH 071/111]

---
 doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn

diff --git a/doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn b/doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn
new file mode 100644
index 0000000000..7a579f8fe6
--- /dev/null
+++ b/doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn
@@ -0,0 +1 @@
+When the test suite cannot be compiled, the build just fails silenty. This means that in automated builds there is no easy way to ensure that the generated binaries have passed the test suite, because it may not even have been run! IMHO, "make test" should fail (i.e. return a non-zero exit code) when it can't succeeed.

From f82da1d9dca0712cdd87e3fc0ed8a2c2e2440228 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 27 Aug 2011 07:08:15 -0400
Subject: [PATCH 072/111] show a message if asked to get something from the web
 that is not there

---
 Remote/Web.hs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Remote/Web.hs b/Remote/Web.hs
index 5bc6a204b9..0e4b757673 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -86,8 +86,12 @@ setUrl key url status = do
 	logChange g key webUUID (if null us then InfoMissing else InfoPresent)
 
 downloadKey :: Key -> FilePath -> Annex Bool
-downloadKey key file = iter =<< getUrls key
+downloadKey key file = get =<< getUrls key
 	where
+		get [] = do
+			warning "no known url"
+			return False
+		get a = iter a
 		iter [] = return False
 		iter (url:urls) = do
 			ok <- Url.download url file

From 6e750764b7d30d9cb0684cdaadd79ec091a4fda6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 27 Aug 2011 12:31:50 -0400
Subject: [PATCH 073/111] The wget command will now be used in preference to
 curl, if available.

Got tired of curl's various ugly progress bars.
---
 Utility.hs       | 12 +++++++++++-
 Utility/Url.hs   | 24 +++++++++++++++++-------
 debian/changelog |  1 +
 debian/control   |  2 +-
 doc/install.mdwn |  2 +-
 5 files changed, 31 insertions(+), 10 deletions(-)

diff --git a/Utility.hs b/Utility.hs
index 788dc41030..451b1b44fc 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -15,7 +15,8 @@ module Utility (
 	dirContains,
 	dirContents,
 	myHomeDir,
-	catchBool
+	catchBool,
+	inPath
 ) where
 
 import IO (bracket)
@@ -94,3 +95,12 @@ myHomeDir = do
 {- Catches IO errors and returns a Bool -}
 catchBool :: IO Bool -> IO Bool
 catchBool = flip catch (const $ return False)
+
+{- Checks if a command is available in PATH. -}
+inPath :: String -> IO Bool
+inPath command = search =<< getSearchPath
+	where
+		search [] = return False
+		search (d:ds) = do
+			e <- doesFileExist $ d  command
+			if e then return True else search ds
diff --git a/Utility/Url.hs b/Utility/Url.hs
index f678720ed8..6ddeecc14f 100644
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -20,6 +20,7 @@ import Network.URI
 import Types
 import Messages
 import Utility.SafeCommand
+import Utility
 
 type URLString = String
 
@@ -35,15 +36,24 @@ exists url =
 				_ -> return False
 
 {- Used to download large files, such as the contents of keys.
- - Uses curl program for its progress bar. -}
+ - Uses wget or curl program for its progress bar. (Wget has a better one,
+ - so is preferred.) -}
 download :: URLString -> FilePath -> Annex Bool
 download url file = do
-	showOutput -- make way for curl progress bar
-	-- Uses the -# progress display, because the normal one is very
-	-- confusing when resuming, showing the remainder to download
-	-- as the whole file, and not indicating how much percent was
-	-- downloaded before the resume.
-	liftIO $ boolSystem "curl" [Params "-L -C - -# -o", File file, File url]
+	showOutput -- make way for program's progress bar
+	e <- liftIO $ inPath "wget"
+	if e
+		then
+			liftIO $ boolSystem "wget"
+				[Params "-c -O", File file, File url]
+		else
+			-- Uses the -# progress display, because the normal
+			-- one is very confusing when resuming, showing
+			-- the remainder to download as the whole file,
+			-- and not indicating how much percent was
+			-- downloaded before the resume.
+			liftIO $ boolSystem "curl" 
+				[Params "-L -C - -# -o", File file, File url]
 
 {- Downloads a small file. -}
 get :: URLString -> IO String
diff --git a/debian/changelog b/debian/changelog
index bb06e1fba9..54fdbd835c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (3.20110820) UNRELEASED; urgency=low
 
   * Set EMAIL when running test suite so that git does not need to be
     configured first. Closes: #638998
+  * The wget command will now be used in preference to curl, if available.
 
  -- Joey Hess   Tue, 23 Aug 2011 13:41:01 -0400
 
diff --git a/debian/control b/debian/control
index 975faf5ea5..63488dc68d 100644
--- a/debian/control
+++ b/debian/control
@@ -31,7 +31,7 @@ Depends: ${misc:Depends}, ${shlibs:Depends},
 	git | git-core,
 	uuid,
 	rsync,
-	curl,
+	wget | curl,
 	openssh-client
 Suggests: graphviz, bup, gnupg
 Description: manage files with git, without checking their contents into git
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 49ddd913f0..ac521da187 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -34,7 +34,7 @@ To build and use git-annex, you will need:
     (or `uuidgen` from util-linux)
   * [xargs](http://savannah.gnu.org/projects/findutils/)
   * [rsync](http://rsync.samba.org/)
-  * [curl](http://http://curl.haxx.se/) (optional, but recommended)
+  * [wget](http://www.gnu.org/software/wget/) or [curl](http://http://curl.haxx.se/) (optional, but recommended)
   * [sha1sum](ftp://ftp.gnu.org/gnu/coreutils/) (optional, but recommended;
     a sha1 command will also do)
   * [gpg](http://gnupg.org/) (optional; needed for encryption)

From b26ee162f33858aa4ef82fbe3d56eecc00590755 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 27 Aug 2011 12:34:02 -0400
Subject: [PATCH 074/111] guess not everyone configures their shell to show
 nonzero exit codes..

---
 Makefile                                              | 4 +++-
 doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn | 2 ++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index fa087c1ae2..20d1347140 100644
--- a/Makefile
+++ b/Makefile
@@ -53,7 +53,9 @@ test: $(bins)
 	@if ! $(GHCMAKE) -O0 test; then \
 		echo "** not running test suite" >&2; \
 	else \
-		./test; \
+		if ! ./test; then \
+			echo "** test suite failed!" >&2; \
+		fi; \
 	fi
 
 testcoverage: $(bins)
diff --git a/doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn b/doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn
index 7a579f8fe6..2f486ad652 100644
--- a/doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn
+++ b/doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn
@@ -1 +1,3 @@
 When the test suite cannot be compiled, the build just fails silenty. This means that in automated builds there is no easy way to ensure that the generated binaries have passed the test suite, because it may not even have been run! IMHO, "make test" should fail (i.e. return a non-zero exit code) when it can't succeeed.
+
+> Ok, fixed. --[[Joey]] [[done]]

From 999d5df90b013a7cc8a390c940785118400faf8a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 28 Aug 2011 15:46:49 -0400
Subject: [PATCH 075/111] factor out firstM and anyM

Control.Monad.Loops has these, but has no Debian package yet.
---
 Remote/Web.hs |  7 ++-----
 Utility.hs    | 28 ++++++++++++++++++++++------
 2 files changed, 24 insertions(+), 11 deletions(-)

diff --git a/Remote/Web.hs b/Remote/Web.hs
index 0e4b757673..3695bb1ab3 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -24,6 +24,7 @@ import Config
 import PresenceLog
 import LocationLog
 import Locations
+import Utility
 import qualified Utility.Url as Url
 
 type URLString = String
@@ -91,11 +92,7 @@ downloadKey key file = get =<< getUrls key
 		get [] = do
 			warning "no known url"
 			return False
-		get a = iter a
-		iter [] = return False
-		iter (url:urls) = do
-			ok <- Url.download url file
-			if ok then return ok else iter urls
+		get urls = anyM (`Url.download` file) urls
 
 uploadKey :: Key -> Annex Bool
 uploadKey _ = do
diff --git a/Utility.hs b/Utility.hs
index 451b1b44fc..ce17363488 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -16,7 +16,9 @@ module Utility (
 	dirContents,
 	myHomeDir,
 	catchBool,
-	inPath
+	inPath,
+	firstM,
+	anyM
 ) where
 
 import IO (bracket)
@@ -29,6 +31,8 @@ import System.FilePath
 import System.Directory
 import Foreign (complement)
 import Utility.Path
+import Data.Maybe
+import Control.Monad (liftM)
 
 {- A version of hgetContents that is not lazy. Ensures file is 
  - all read before it gets closed. -}
@@ -96,11 +100,23 @@ myHomeDir = do
 catchBool :: IO Bool -> IO Bool
 catchBool = flip catch (const $ return False)
 
+{- Return the first value from a list, if any, satisfying the given
+ - predicate -}
+firstM :: (Monad m) => (a -> m Bool) -> [a] -> m (Maybe a)
+firstM _ [] = return Nothing
+firstM p (x:xs) = do
+	q <- p x
+	if q
+		then return (Just x)
+		else firstM p xs
+
+{- Returns true if any value in the list satisfies the preducate,
+ - stopping once one is found. -}
+anyM :: (Monad m) => (a -> m Bool) -> [a] -> m Bool
+anyM p = liftM isJust . firstM p
+
 {- Checks if a command is available in PATH. -}
 inPath :: String -> IO Bool
-inPath command = search =<< getSearchPath
+inPath command = getSearchPath >>= anyM indir
 	where
-		search [] = return False
-		search (d:ds) = do
-			e <- doesFileExist $ d  command
-			if e then return True else search ds
+		indir d = doesFileExist $ d  command

From bbba6c19bd03f2b4a4ce8a38a2423c794826b1c5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 28 Aug 2011 16:28:38 -0400
Subject: [PATCH 076/111] update documentation for new, neutered key-value
 backends

Backends are now only used to generate keys (and check them); they
are not arbitrary key-value stores for data, because it turned out such
a store is better modeled as a special remote. Updated docs to not
imply backends do more than they do now.

Sometimes I'm tempted to rename "backend" to "keytype" or something,
which would really be more clear. But it would be an annoying transition
for users, with annex.backends etc.
---
 doc/backends.mdwn                             | 22 +++++-----------
 doc/copies.mdwn                               |  6 ++---
 doc/git-annex.mdwn                            | 26 +++++++++----------
 .../fsck:_verifying_your_data.mdwn            | 10 +++----
 doc/walkthrough/unused_data.mdwn              |  2 +-
 5 files changed, 29 insertions(+), 37 deletions(-)

diff --git a/doc/backends.mdwn b/doc/backends.mdwn
index 9e698032d8..f2f1c5580c 100644
--- a/doc/backends.mdwn
+++ b/doc/backends.mdwn
@@ -1,24 +1,16 @@
-git-annex uses a key-value abstraction layer to allow file contents to be
-stored in different ways. In theory, any key-value storage system could be
-used to store file contents.
-
 When a file is annexed, a key is generated from its content and/or metadata.
 The file checked into git symlinks to the key. This key can later be used
 to retrieve the file's content (its value).
 
-Multiple pluggable backends are supported, and a single repository
-can use different backends for different files.
+Multiple pluggable key-value backends are supported, and a single repository
+can use different ones for different files.
 
-These backends can transfer file contents between configured git remotes.
-It's also possible to use [[special_remotes]], such as Amazon S3 with
-these backends.
-
-* `WORM` ("Write Once, Read Many") This backend assumes that any file with
-  the same basename, size, and modification time has the same content. So with
-  this backend, files can be moved around, but should never be added to
+* `WORM` ("Write Once, Read Many") This assumes that any file with
+  the same basename, size, and modification time has the same content. So
+  files can be moved around, but should never be added to
   or changed. This is the default, and the least expensive backend.
-* `SHA1` -- This backend uses a key based on a sha1 checksum. This backend
-  allows modifications of files to be tracked. Its need to generate checksums
+* `SHA1` -- This uses a key based on a sha1 checksum. This allows
+  modifications of files to be tracked. Its need to generate checksums
   can make it slower for large files.
 * `SHA512`, `SHA384`, `SHA256`, `SHA224` -- Like SHA1, but larger
   checksums. Mostly useful for the very paranoid, or anyone who is
diff --git a/doc/copies.mdwn b/doc/copies.mdwn
index 16eba19c81..93cbd8ea80 100644
--- a/doc/copies.mdwn
+++ b/doc/copies.mdwn
@@ -1,8 +1,8 @@
-The WORM and SHA1 key-value [[backends]] store data inside
-your git repository's `.git` directory, not in some external data store.
+Annexed data is stored inside  your git repository's `.git/annex` directory.
+Some [[special_remotes]] can store annexed data elsewhere.
 
 It's important that data not get lost by an ill-considered `git annex drop`
-command.  So, then using those backends, git-annex can be configured to try
+command.  So, git-annex can be configured to try
 to keep N copies of a file's content available across all repositories. 
 (Although [[untrusted_repositories|trust]] don't count toward this total.)
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index a262d465fd..52599611ed 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -72,15 +72,15 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * get [path ...]
 
-  Makes the content of annexed files available in this repository. Depending
-  on the backend used, this will involve copying them from another repository,
-  or downloading them, or transferring them from some kind of key-value store.
+  Makes the content of annexed files available in this repository. This
+  will involve copying them from another repository, or downloading them,
+  or transferring them from some kind of key-value store.
 
 * drop [path ...]
 
   Drops the content of annexed files from this repository. 
 
-  git-annex may refuse to drop content if the backend does not think
+  git-annex may refuse to drop content if it does not think
   it is safe to do so, typically because of the setting of annex.numcopies.
 
 * move [path ...]
@@ -207,14 +207,14 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * migrate [path ...]
 
-  Changes the specified annexed files to store their content in the
-  default backend (or the one specified with --backend). Only files whose
-  content is currently available are migrated.
+  Changes the specified annexed files to use the default key-value backend
+  (or the one specified with --backend). Only files whose content
+  is currently available are migrated.
 
-  Note that the content is not removed from the backend it was previously in.
-  Use `git annex unused` to find and remove such content.
+  Note that the content is also still available using the old key after
+  migration. Use `git annex unused` to find and remove the old key.
 
-  Normally, nothing will be done to files already in the backend.
+  Normally, nothing will be done to files already using the new backend.
   However, if a backend changes the information it uses to construct a key,
   this can also be used to migrate files to use the new key format.
 
@@ -293,7 +293,7 @@ Many git-annex commands will stage changes for later `git commit` by you.
 * fromkey file
 
   This plumbing-level command can be used to manually set up a file
-  to link to a specified key in the key-value backend.
+  in the git repository to link to a specified key.
 
 * dropkey [key ...]
 
@@ -500,8 +500,8 @@ Here are all the supported configuration settings.
 
 # CONFIGURATION VIA .gitattributes
 
-The backend used when adding a new file to the annex can be configured
-on a per-file-type basis via `.gitattributes` files. In the file,
+The key-value backend used when adding a new file to the annex can be
+configured on a per-file-type basis via `.gitattributes` files. In the file,
 the `annex.backend` attribute can be set to the name of the backend to
 use. For example, this here's how to use the WORM backend by default,
 but the SHA1 backend for ogg files:
diff --git a/doc/walkthrough/fsck:_verifying_your_data.mdwn b/doc/walkthrough/fsck:_verifying_your_data.mdwn
index 7e05469a12..d036332fb3 100644
--- a/doc/walkthrough/fsck:_verifying_your_data.mdwn
+++ b/doc/walkthrough/fsck:_verifying_your_data.mdwn
@@ -1,8 +1,8 @@
-You can use the fsck subcommand to check for problems in your data.
-What can be checked depends on the [[backend|backends]] you've used to store
-the data. For example, when you use the SHA1 backend, fsck will verify that
-the checksums of your files are good. Fsck also checks that the annex.numcopies
-setting is satisfied for all files.
+You can use the fsck subcommand to check for problems in your data. What
+can be checked depends on the key-value [[backend|backends]] you've used
+for the data. For example, when you use the SHA1 backend, fsck will verify
+that the checksums of your files are good. Fsck also checks that the
+annex.numcopies setting is satisfied for all files.
 
 	# git annex fsck
 	fsck some_file (checksum...) ok
diff --git a/doc/walkthrough/unused_data.mdwn b/doc/walkthrough/unused_data.mdwn
index fb84193034..e142b576c0 100644
--- a/doc/walkthrough/unused_data.mdwn
+++ b/doc/walkthrough/unused_data.mdwn
@@ -2,7 +2,7 @@ It's possible for data to accumulate in the annex that no files point to
 anymore. One way it can happen is if you `git rm` a file without 
 first calling `git annex drop`. And, when you modify an annexed file, the old
 content of the file remains in the annex. Another way is when migrating
-between backends.
+between key-value [[backends|backend]].
 
 This might be historical data you want to preserve, so git-annex defaults to
 preserving it. So from time to time, you may want to check for such data and

From b6758746f6433a46ac14bad58f9f6e0220a42e3e Mon Sep 17 00:00:00 2001
From: cwg 
Date: Mon, 29 Aug 2011 12:35:19 +0000
Subject: [PATCH 077/111]

---
 doc/forum/advantages_of_SHA__42___over_WORM.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 doc/forum/advantages_of_SHA__42___over_WORM.mdwn

diff --git a/doc/forum/advantages_of_SHA__42___over_WORM.mdwn b/doc/forum/advantages_of_SHA__42___over_WORM.mdwn
new file mode 100644
index 0000000000..b959db550c
--- /dev/null
+++ b/doc/forum/advantages_of_SHA__42___over_WORM.mdwn
@@ -0,0 +1,6 @@
+Thanks for creating git-annex.
+
+I am confused about the advantages of the SHA* backends over WORM.  The "backends" page in this wiki says that with WORM files "can be moved around, but should never be added to or changed".  But I don't see any difference to SHA* files as long as the premise of WORM that "any file with the same basename, size, and modification time has the same content" is true.
+
+Using "git annex unlock", WORM files can be modified in the same way as SHA* files.
+If the storage I use is dependable (i.e. I don't need SHA checksums for detection of corruption), and I don't need to optimize for the case that the modification date of a file is changed but the contents stay the same, is there actually any advantage in using SHA*?

From eb4607aafc193073468da09a3cac5d54f6c16735 Mon Sep 17 00:00:00 2001
From: cwg 
Date: Mon, 29 Aug 2011 12:37:23 +0000
Subject: [PATCH 078/111]

---
 doc/forum/advantages_of_SHA__42___over_WORM.mdwn | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/doc/forum/advantages_of_SHA__42___over_WORM.mdwn b/doc/forum/advantages_of_SHA__42___over_WORM.mdwn
index b959db550c..5b544593f5 100644
--- a/doc/forum/advantages_of_SHA__42___over_WORM.mdwn
+++ b/doc/forum/advantages_of_SHA__42___over_WORM.mdwn
@@ -1,6 +1,5 @@
 Thanks for creating git-annex.
 
-I am confused about the advantages of the SHA* backends over WORM.  The "backends" page in this wiki says that with WORM files "can be moved around, but should never be added to or changed".  But I don't see any difference to SHA* files as long as the premise of WORM that "any file with the same basename, size, and modification time has the same content" is true.
+I am confused about the advantages of the SHA* backends over WORM.  The "backends" page in this wiki says that with WORM, files "can be moved around, but should never be added to or changed".  But I don't see any difference to SHA* files as long as the premise of WORM that "any file with the same basename, size, and modification time has the same content" is true.  Using "git annex unlock", WORM files can be modified in the same way as SHA* files.
 
-Using "git annex unlock", WORM files can be modified in the same way as SHA* files.
-If the storage I use is dependable (i.e. I don't need SHA checksums for detection of corruption), and I don't need to optimize for the case that the modification date of a file is changed but the contents stay the same, is there actually any advantage in using SHA*?
+If the storage I use is dependable (i.e. I don't need SHA checksums for detection of corruption), and I don't need to optimize for the case that the modification date of a file is changed but the contents stay the same, and if it is unlikely that several files will be identical, is there actually any advantage in using SHA*?

From 2a76ad41673968793a479f79ed33ba83a0b10773 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 29 Aug 2011 16:10:38 +0000
Subject: [PATCH 079/111] Added a comment

---
 .../comment_1_96c354cac4b5ce5cf6664943bc84db1d._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/advantages_of_SHA__42___over_WORM/comment_1_96c354cac4b5ce5cf6664943bc84db1d._comment

diff --git a/doc/forum/advantages_of_SHA__42___over_WORM/comment_1_96c354cac4b5ce5cf6664943bc84db1d._comment b/doc/forum/advantages_of_SHA__42___over_WORM/comment_1_96c354cac4b5ce5cf6664943bc84db1d._comment
new file mode 100644
index 0000000000..218027ca53
--- /dev/null
+++ b/doc/forum/advantages_of_SHA__42___over_WORM/comment_1_96c354cac4b5ce5cf6664943bc84db1d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-08-29T16:10:38Z"
+ content="""
+You're right -- as long as nothing changes a file without letting the modification time update, editing WORM files is safe.
+"""]]

From 025e66e3d35d89fc3c3dd2c81e2f618941130a26 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 29 Aug 2011 12:08:54 -0400
Subject: [PATCH 080/111] update to not overstate the danger or WORM

---
 doc/backends.mdwn | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/backends.mdwn b/doc/backends.mdwn
index f2f1c5580c..ebcdedc2a7 100644
--- a/doc/backends.mdwn
+++ b/doc/backends.mdwn
@@ -6,11 +6,11 @@ Multiple pluggable key-value backends are supported, and a single repository
 can use different ones for different files.
 
 * `WORM` ("Write Once, Read Many") This assumes that any file with
-  the same basename, size, and modification time has the same content. So
-  files can be moved around, but should never be added to
-  or changed. This is the default, and the least expensive backend.
+  the same basename, size, and modification time has the same content.
+  This is the default, and the least expensive backend.
 * `SHA1` -- This uses a key based on a sha1 checksum. This allows
-  modifications of files to be tracked. Its need to generate checksums
+  verifying that the file content is right, and can avoid duplicates of
+  files with the same content. Its need to generate checksums
   can make it slower for large files.
 * `SHA512`, `SHA384`, `SHA256`, `SHA224` -- Like SHA1, but larger
   checksums. Mostly useful for the very paranoid, or anyone who is
@@ -28,7 +28,7 @@ files, the `.gitattributes` file can be used. The `annex.backend`
 attribute can be set to the name of the backend to use for matching files.
 
 For example, to use the SHA1 backend for sound files, which tend to be
-smallish and might be modified over time, you could set in
+smallish and might be modified or copied over time, you could set in
 `.gitattributes`:
 
 	*.mp3 annex.backend=SHA1

From 676c467801051e71ea7668615985301f26ab0f3e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 29 Aug 2011 12:49:38 -0400
Subject: [PATCH 081/111] close

---
 ...ude_repo_description_and__47__or_UUID_in_commit_message.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn b/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn
index 9ca61bff55..be7e2dacc8 100644
--- a/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn
+++ b/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn
@@ -9,3 +9,5 @@ I'm not sure that the above suggestion is going down a path that really
 makes sense. If you want a list of repository UUIDs and descriptions,
 it's there in machine-usable form in `.git-annex/uuid.log`, there is no
 need to try to pull this info out of git commit messages. --[[Joey]]
+
+[[done]]

From b2c5639dcc53f6e734643878f9696405fa6cae64 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 29 Aug 2011 13:29:39 -0400
Subject: [PATCH 082/111] update

---
 doc/todo/smudge.mdwn | 46 +++++++++++++++++++++++++++++++++-----------
 1 file changed, 35 insertions(+), 11 deletions(-)

diff --git a/doc/todo/smudge.mdwn b/doc/todo/smudge.mdwn
index c51662b282..f78b215ac4 100644
--- a/doc/todo/smudge.mdwn
+++ b/doc/todo/smudge.mdwn
@@ -19,6 +19,26 @@ add` files, and just being able to use `git add` or `git commit -a`,
 and have it use git-annex when .gitattributes says to. Also, annexed
 files can be directly modified without having to `git annex unlock`.
 
+### design
+
+In .gitattributes, the user would put something like "* filter=git-annex".
+This way they could control which files are annexed vs added normally.
+
+(git-annex could have further controls to allow eg, passing small files
+through to regular processing. At least .gitattributes is a special case,
+it should never be annexed...)
+
+For files not configured this way, git-annex could continue to use
+its symlink method -- this would preserve backwards compatability,
+and even allow mixing the two methods in a repo as desired.
+
+To find files in the repository that are annexed, git-annex would do
+`ls-files` as now, but would check if found files have the appropriate
+filter, rather than the current symlink checks. To determine the key
+of a file, rather than reading its symlink, git-annex would need to
+look up the git blob associated with the file -- this can be done
+efficiently using the existing code in `Branch.catFile`.
+
 ### efficiency
 
 The trick is doing it efficiently. Since git a2b665d, v1.7.4.1,
@@ -30,12 +50,16 @@ This avoids it needing to read all the current file content from stdin
 when doing eg, a git status or git commit. Instead it is passed the
 filename that git is operating on, in the working directory.
 
+(The smudge script can also be provided a filename with %f, but it
+cannot directly write to the file or git gets unhappy.)
+
 So, WORM could just look at that file and easily tell if it is one
 it already knows (same mtime and size). If so, it can short-circuit and
 do nothing, file content is already cached.
 
 SHA1 has a harder job. Would not want to re-sha1 the file every time,
-probably. So it'd need a cache of file stat info, mapped to known objects.
+probably. So it'd need a local cache of file stat info, mapped to known
+objects.
 
 ### dealing with partial content availability
 
@@ -59,9 +83,10 @@ huge-smudge:
 
 #!/bin/sh
 read sha1
+file="$1"
 echo "smudging $sha1" >&2
 if [ -e ~/$sha1 ]; then
-	cat ~/$sha1
+	cat ~/$sha1 # possibly expensive copy here
 else
 	echo "$sha1 not available"
 fi
@@ -71,16 +96,15 @@ huge-clean:
 
 
 #!/bin/sh
-cat >temp
-if grep -q 'not available' temp; then
-	awk '{print $1}' temp # provide what we would if the content were avail!
-	rm temp
+temp="$1"
+if grep -q 'not available' "$temp"; then
+	awk '{print $1}' "$temp" # provide what we would if the content were avail!
 	exit 0
 fi
-sha1=`sha1sum temp | cut -d' ' -f1`
+sha1=`sha1sum "$temp" | cut -d' ' -f1`
 echo "cleaning $sha1" >&2
-ls -l temp >&2
-mv temp ~/$sha1
+ls -l "$temp" >&2
+ln -f "$temp" ~/$sha1 # can't delete temp file
 echo $sha1
 

 
@@ -94,6 +118,6 @@ in .git/config:
 
 
 [filter "huge"]
-        clean = huge-clean
-        smudge = huge-smudge
+        clean = huge-clean %f
+        smudge = huge-smudge %f
 

From d1154d0837f4dac18771a6a5c3ab96de4d5fcd90 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 29 Aug 2011 14:13:38 -0400
Subject: [PATCH 083/111] init: Make description an optional parameter.

---
 Command/Init.hs    | 6 ++----
 debian/changelog   | 1 +
 doc/git-annex.mdwn | 4 +---
 3 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/Command/Init.hs b/Command/Init.hs
index 0191060511..2abe4c6615 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -7,7 +7,7 @@
 
 module Command.Init where
 
-import Control.Monad (when)
+import Control.Monad
 
 import Command
 import qualified Annex
@@ -17,15 +17,13 @@ import Init
 	
 command :: [Command]
 command = [standaloneCommand "init" paramDesc seek
-		"initialize git-annex with repository description"]
+		"initialize git-annex"]
 
 seek :: [CommandSeek]
 seek = [withWords start]
 
 start :: CommandStartWords
 start ws = do
-	when (null description) $
-		error "please specify a description of this repository\n"
 	showStart "init" description
 	next $ perform description
 	where
diff --git a/debian/changelog b/debian/changelog
index 54fdbd835c..7803c9a62a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,7 @@ git-annex (3.20110820) UNRELEASED; urgency=low
   * Set EMAIL when running test suite so that git does not need to be
     configured first. Closes: #638998
   * The wget command will now be used in preference to curl, if available.
+  * init: Make description an optional parameter.
 
  -- Joey Hess   Tue, 23 Aug 2011 13:41:01 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 52599611ed..e7ac9adf7a 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -119,9 +119,7 @@ Many git-annex commands will stage changes for later `git commit` by you.
   Use this to undo an unlock command if you don't want to modify
   the files, or have made modifications you want to discard.
 
-* init description
-
-  Initializes git-annex with a description of the git repository.
+* init [description]
 
   Until a repository (or one of its remotes) has been initialized,
   git-annex will refuse to operate on it, to avoid accidentially

From cd199e442f31743ee7ddc7c545f390533ed30a9d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 29 Aug 2011 16:31:47 -0400
Subject: [PATCH 084/111] update; showstopper issue with current git

developed a patch for git, we'll see if they like it..
---
 doc/todo/smudge.mdwn | 49 +++++++++++++++++++++++++++++---------------
 1 file changed, 32 insertions(+), 17 deletions(-)

diff --git a/doc/todo/smudge.mdwn b/doc/todo/smudge.mdwn
index f78b215ac4..2f5d21d7e1 100644
--- a/doc/todo/smudge.mdwn
+++ b/doc/todo/smudge.mdwn
@@ -41,18 +41,17 @@ efficiently using the existing code in `Branch.catFile`.
 
 ### efficiency
 
+#### clean
+
 The trick is doing it efficiently. Since git a2b665d, v1.7.4.1,
 something like this works to provide a filename to the clean script:
 
 	git config --global filter.huge.clean huge-clean %f
 
-This avoids it needing to read all the current file content from stdin
+This could avoid it needing to read all the current file content from stdin
 when doing eg, a git status or git commit. Instead it is passed the
 filename that git is operating on, in the working directory.
 
-(The smudge script can also be provided a filename with %f, but it
-cannot directly write to the file or git gets unhappy.)
-
 So, WORM could just look at that file and easily tell if it is one
 it already knows (same mtime and size). If so, it can short-circuit and
 do nothing, file content is already cached.
@@ -61,6 +60,21 @@ SHA1 has a harder job. Would not want to re-sha1 the file every time,
 probably. So it'd need a local cache of file stat info, mapped to known
 objects.
 
+But: Even with %f, git actually passes the full file content to the clean
+filter, and if it fails to consume it all, it will crash (may only happen
+if the file is larger than some chunk size; tried with 500 mb file and 
+saw a SIGPIPE.) This means unnecessary works needs to be done, 
+and it slows down *everything*, from `git status` to `git commit`.
+**showstopper** I have sent a patch to the git mailing list to address
+this.
+
+#### smudge
+
+The smudge script can also be provided a filename with %f, but it
+cannot directly write to the file or git gets unhappy.
+
+
+
 ### dealing with partial content availability
 
 The smudge filter cannot be allowed to fail, that leaves the tree and
@@ -82,13 +96,13 @@ huge-smudge:
 
 
 #!/bin/sh
-read sha1
+read f
 file="$1"
-echo "smudging $sha1" >&2
-if [ -e ~/$sha1 ]; then
-	cat ~/$sha1 # possibly expensive copy here
+echo "smudging $f" >&2
+if [ -e ~/$f ]; then
+	cat ~/$f # possibly expensive copy here
 else
-	echo "$sha1 not available"
+	echo "$f not available"
 fi
 

 
@@ -96,16 +110,17 @@ huge-clean:
 
 
 #!/bin/sh
-temp="$1"
-if grep -q 'not available' "$temp"; then
-	awk '{print $1}' "$temp" # provide what we would if the content were avail!
+file="$1"
+# in real life, this should be done more efficiently, not trying to read
+# the whole file content!
+if grep -q 'not available' "$file"; then
+	awk '{print $1}' "$file" # provide what we would if the content were avail!
 	exit 0
 fi
-sha1=`sha1sum "$temp" | cut -d' ' -f1`
-echo "cleaning $sha1" >&2
-ls -l "$temp" >&2
-ln -f "$temp" ~/$sha1 # can't delete temp file
-echo $sha1
+echo "cleaning $file" >&2
+ls -l "$file" >&2
+ln -f "$file" ~/$file # can't delete temp file
+echo $file
 

 
 .gitattributes:

From 5ef11350aafe2aec48f66799743423a9c6e42556 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 29 Aug 2011 16:41:47 -0400
Subject: [PATCH 085/111] link to patch

---
 doc/todo/smudge.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/todo/smudge.mdwn b/doc/todo/smudge.mdwn
index 2f5d21d7e1..7d8f9a136d 100644
--- a/doc/todo/smudge.mdwn
+++ b/doc/todo/smudge.mdwn
@@ -66,7 +66,7 @@ if the file is larger than some chunk size; tried with 500 mb file and
 saw a SIGPIPE.) This means unnecessary works needs to be done, 
 and it slows down *everything*, from `git status` to `git commit`.
 **showstopper** I have sent a patch to the git mailing list to address
-this.
+this. 
 
 #### smudge
 

From 9e135a6ee2b6b7ac31bbf76d0d590a0415f9f512 Mon Sep 17 00:00:00 2001
From: "http://www.schleptet.net/~cfm/" 
Date: Tue, 30 Aug 2011 14:31:38 +0000
Subject: [PATCH 086/111] Added a comment

---
 ...ment_1_0a1760bf0db1f1ba89bdb4c62032f631._comment | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/install/OSX/comment_1_0a1760bf0db1f1ba89bdb4c62032f631._comment

diff --git a/doc/install/OSX/comment_1_0a1760bf0db1f1ba89bdb4c62032f631._comment b/doc/install/OSX/comment_1_0a1760bf0db1f1ba89bdb4c62032f631._comment
new file mode 100644
index 0000000000..1148a87cab
--- /dev/null
+++ b/doc/install/OSX/comment_1_0a1760bf0db1f1ba89bdb4c62032f631._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://www.schleptet.net/~cfm/"
+ ip="64.30.148.100"
+ subject="comment 1"
+ date="2011-08-30T14:31:36Z"
+ content="""
+You can also use Homebrew instead of MacPorts.  Homebrew's `haskell-platform` is up-to-date, too:
+
+    brew install haskell-platform git ossp-uuid md5sha1sum coreutils pcre
+    ln -s /usr/local/include/pcre.h /usr/include/pcre.h
+
+As of this writing, however, Homebrew's `md5sha1sum` has a broken mirror.  I wound up getting that from MacPorts anyway.
+"""]]

From a64c16bf7d6cf3ba5295a23496e10ed70c1656dc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 30 Aug 2011 13:23:21 -0400
Subject: [PATCH 087/111] tweak

---
 Backend.hs           | 7 +++----
 Utility/DataUnits.hs | 3 +--
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index 0bb9f4b575..75327de80b 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -122,8 +122,7 @@ lookupBackendName s = fromMaybe unknown $ maybeLookupBackendName s
 	where
 		unknown = error $ "unknown backend " ++ s
 maybeLookupBackendName :: String -> Maybe (Backend Annex)
-maybeLookupBackendName s =
-	if 1 /= length matches
-		then Nothing
-		else Just $ head matches
+maybeLookupBackendName s
+	| length matches == 1 = Just $ head matches
+	| otherwise = Nothing
 	where matches = filter (\b -> s == B.name b) list
diff --git a/Utility/DataUnits.hs b/Utility/DataUnits.hs
index f2bc333ea0..0baa5dd896 100644
--- a/Utility/DataUnits.hs
+++ b/Utility/DataUnits.hs
@@ -137,8 +137,7 @@ compareSizes units abbrev old new
 {- Parses strings like "10 kilobytes" or "0.5tb". -}
 readSize :: [Unit] -> String -> Maybe ByteSize
 readSize units input
-	| null parsednum = Nothing
-	| null parsedunit = Nothing
+	| null parsednum || null parsedunit = Nothing
 	| otherwise = Just $ round $ number * fromIntegral multiplier
 	where
 		(number, rest) = head parsednum

From b96443364e80720172638f0fcd9caa5b9948555d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 30 Aug 2011 13:29:07 -0400
Subject: [PATCH 088/111] smudge update: Not practical.

---
 doc/todo/smudge.mdwn | 38 +++++++++++++++++++++++++++++---------
 1 file changed, 29 insertions(+), 9 deletions(-)

diff --git a/doc/todo/smudge.mdwn b/doc/todo/smudge.mdwn
index 7d8f9a136d..55de5f5789 100644
--- a/doc/todo/smudge.mdwn
+++ b/doc/todo/smudge.mdwn
@@ -1,5 +1,18 @@
 git-annex should use smudge/clean filters.
 
+----
+
+Update: Currently, this does not look likely to work. In particular,
+the clean filter needs to consume all stdin from git, which consists of the
+entire content of the file. It cannot optimise by directly accessing
+the file in the repository, because git may be cleaning a different
+version of the file during a merge. 
+
+So every `git status` would need to read the entire content of all
+available files, and checksum them, which is too expensive.
+
+----
+
 The clean filter is run when files are staged for commit. So a user could copy
 any file into the annex, git add it, and git-annex's clean filter causes
 the file's key to be staged, while its value is added to the annex.
@@ -8,7 +21,7 @@ The smudge filter is run when files are checked out. Since git annex
 repos have partial content, this would not git annex get the file content.
 Instead, if the content is not currently available, it would need to do
 something like return empty file content. (Sadly, it cannot create a
-symlink, as git still wants to write the file afterwards.
+symlink, as git still wants to write the file afterwards.)
 
 So the nice current behavior of unavailable files being clearly missing due
 to dangling symlinks, would be lost when using smudge/clean filters.
@@ -39,7 +52,13 @@ of a file, rather than reading its symlink, git-annex would need to
 look up the git blob associated with the file -- this can be done
 efficiently using the existing code in `Branch.catFile`.
 
-### efficiency
+The clean filter would inject the file's content into the annex, and hard
+link from the annex to the file. Avoiding duplication of data.
+
+The smudge filter can't do that, so to avoid duplication of data, it
+might always create an empty file. To get the content, `git annex get`
+could be used (which would hard link it). A `post-checkout` hook might
+be used to set up hard links for all currently available content.
 
 #### clean
 
@@ -51,6 +70,8 @@ something like this works to provide a filename to the clean script:
 This could avoid it needing to read all the current file content from stdin
 when doing eg, a git status or git commit. Instead it is passed the
 filename that git is operating on, in the working directory.
+(Update: No, doesn't work; git may be cleaning a different file content
+than is currently on disk, and git requires all stdin be consumed too.)
 
 So, WORM could just look at that file and easily tell if it is one
 it already knows (same mtime and size). If so, it can short-circuit and
@@ -66,15 +87,14 @@ if the file is larger than some chunk size; tried with 500 mb file and
 saw a SIGPIPE.) This means unnecessary works needs to be done, 
 and it slows down *everything*, from `git status` to `git commit`.
 **showstopper** I have sent a patch to the git mailing list to address
-this. 
+this.  (Update: apparently
+can't be fixed.)
 
 #### smudge
 
 The smudge script can also be provided a filename with %f, but it
 cannot directly write to the file or git gets unhappy.
 
-
-
 ### dealing with partial content availability
 
 The smudge filter cannot be allowed to fail, that leaves the tree and
@@ -111,15 +131,15 @@ huge-clean:
 
 #!/bin/sh
 file="$1"
+cat >/tmp/file
 # in real life, this should be done more efficiently, not trying to read
 # the whole file content!
-if grep -q 'not available' "$file"; then
-	awk '{print $1}' "$file" # provide what we would if the content were avail!
+if grep -q 'not available' /tmp/file; then
+	awk '{print $1}' /tmp/file # provide what we would if the content were avail!
 	exit 0
 fi
 echo "cleaning $file" >&2
-ls -l "$file" >&2
-ln -f "$file" ~/$file # can't delete temp file
+# XXX store file content here
 echo $file
 

 

From ea7b1828d48a5dce42393d252f4bbda5cba10d24 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 30 Aug 2011 15:16:34 -0400
Subject: [PATCH 089/111] unused, status: Sped up by avoiding unnecessary stats
 of annexed files.

Statting files returned by dirContents to see if they exist and are regular
files seems pretty useless. This code was originally part of fsck, and
perhaps the idea then was to avoid things returned by dirContents that were
not files. But it's certianly not needed in the current use cases for
getKeysPresent.
---
 Content.hs       | 9 +--------
 debian/changelog | 1 +
 2 files changed, 2 insertions(+), 8 deletions(-)

diff --git a/Content.hs b/Content.hs
index ba99f1330d..1c2475240b 100644
--- a/Content.hs
+++ b/Content.hs
@@ -254,15 +254,8 @@ getKeysPresent' dir = do
 			levela <- dirContents dir
 			levelb <- mapM dirContents levela
 			contents <- mapM dirContents (concat levelb)
-			files <- filterM present (concat contents)
+			let files = concat contents
 			return $ mapMaybe (fileKey . takeFileName) files
-	where
-		present d = do
-			result <- try $
-				getFileStatus $ d  takeFileName d
-			case result of
-				Right s -> return $ isRegularFile s
-				Left _ -> return False
 
 {- Things to do to record changes to content. -}
 saveState :: Annex ()
diff --git a/debian/changelog b/debian/changelog
index 7803c9a62a..c8c60579e2 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,7 @@ git-annex (3.20110820) UNRELEASED; urgency=low
     configured first. Closes: #638998
   * The wget command will now be used in preference to curl, if available.
   * init: Make description an optional parameter.
+  * unused, status: Sped up by avoiding unnecessary stats of annexed files.
 
  -- Joey Hess   Tue, 23 Aug 2011 13:41:01 -0400
 

From baa43cb9c0b38135e136a8ff807c4caeb909b538 Mon Sep 17 00:00:00 2001
From: "http://jefferai.org/" 
Date: Tue, 30 Aug 2011 20:56:44 +0000
Subject: [PATCH 090/111]

---
 doc/bugs/add_script-friendly_output_options.mdwn | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 doc/bugs/add_script-friendly_output_options.mdwn

diff --git a/doc/bugs/add_script-friendly_output_options.mdwn b/doc/bugs/add_script-friendly_output_options.mdwn
new file mode 100644
index 0000000000..efcd54553f
--- /dev/null
+++ b/doc/bugs/add_script-friendly_output_options.mdwn
@@ -0,0 +1,3 @@
+I have a need to use git-annex from a larger program. It'd be great if the information output by some of the commands that is descriptive (for example, whereis) could be sent to stdout in a machine-readable format like (preferably) JSON, or XML. That way I can simply read in the output of the command and use the data directly instead of having to parse it via regexes or other such string manipulation.
+
+This could perhaps be triggered by a --json or --xml flag to the relevant commands.

From f600444ab6d00826cc16c4a5cc67aca84cc1f9e0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 31 Aug 2011 19:13:02 -0400
Subject: [PATCH 091/111] unused --remote: Reduced memory use to 1/4th what was
 used before.

Using a single strictness annotation, in just the right place.
Tried several others, none of which helped and some of which potentially
hurt. This is only the second time I've really had to deal with this in
a year of using haskell, which is, I suppose not that bad.
---
 Command/Unused.hs                          | 5 ++++-
 debian/changelog                           | 1 +
 doc/todo/git-annex_unused_eats_memory.mdwn | 6 ------
 3 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index e7065b3c36..27f5af1b45 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -5,6 +5,8 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
+{-# LANGUAGE BangPatterns #-}
+
 module Command.Unused where
 
 import Control.Monad (filterM, unless, forM_)
@@ -80,7 +82,8 @@ checkRemoteUnused' r = do
 	where
 		isthere k = do
 			us <- keyLocations k
-			return $ uuid `elem` us
+			let !there = uuid `elem` us
+			return there
 		uuid = Remote.uuid r
 
 writeUnusedFile :: FilePath -> [(Int, Key)] -> Annex ()
diff --git a/debian/changelog b/debian/changelog
index c8c60579e2..4ee0b80f2e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,7 @@ git-annex (3.20110820) UNRELEASED; urgency=low
   * The wget command will now be used in preference to curl, if available.
   * init: Make description an optional parameter.
   * unused, status: Sped up by avoiding unnecessary stats of annexed files.
+  * unused --remote: Reduced memory use to 1/4th what was used before.
 
  -- Joey Hess   Tue, 23 Aug 2011 13:41:01 -0400
 
diff --git a/doc/todo/git-annex_unused_eats_memory.mdwn b/doc/todo/git-annex_unused_eats_memory.mdwn
index 6ce7140045..fcb09a1af7 100644
--- a/doc/todo/git-annex_unused_eats_memory.mdwn
+++ b/doc/todo/git-annex_unused_eats_memory.mdwn
@@ -17,9 +17,3 @@ currently present in the repository (possibly using a bloom filter again),
 and that would yield a shortlist of keys that are probably not used. 
 Then scan thru all files in the repo to make sure that none point to keys
 on the shortlist.
-
-----
-
-`git annex unused --from remote` is much worse, using hundreds of mb of
-memory. It has not been profiled at all yet, and can probably be improved
-somewhat by fixing whatever memory leak it (probably) has.

From 55783d886d3300555d4b68ff7323e2365928e059 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 31 Aug 2011 19:50:08 -0400
Subject: [PATCH 092/111] add explanation for why strictness is needed here

---
 Command/Unused.hs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 27f5af1b45..6a62cde5f8 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -80,6 +80,8 @@ checkRemoteUnused' r = do
 		showLongNote $ remoteUnusedMsg r list
 		showLongNote "\n"
 	where
+		{- This should run strictly to avoid the filterM
+		 - building many thunks containing keyLocations data. -}
 		isthere k = do
 			us <- keyLocations k
 			let !there = uuid `elem` us

From 57dd34c6be5dbc01286108fd943ff9e02956e8aa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 1 Sep 2011 13:35:07 -0400
Subject: [PATCH 093/111] generalize quiet flag to output type

This will allow adding other styles of output.
---
 Annex.hs        | 7 +++++--
 Command/Init.hs | 2 --
 Content.hs      | 3 +--
 Messages.hs     | 7 ++++---
 Options.hs      | 6 +++---
 5 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 287aed875e..fac5d27e49 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -10,6 +10,7 @@
 module Annex (
 	Annex,
 	AnnexState(..),
+	OutputType(..),
 	new,
 	run,
 	eval,
@@ -48,7 +49,7 @@ data AnnexState = AnnexState
 	, backends :: [Backend Annex]
 	, remotes :: [Remote Annex]
 	, repoqueue :: Queue
-	, quiet :: Bool
+	, output :: OutputType
 	, force :: Bool
 	, fast :: Bool
 	, branchstate :: BranchState
@@ -63,13 +64,15 @@ data AnnexState = AnnexState
 	, cipher :: Maybe Cipher
 	}
 
+data OutputType = NormalOutput | QuietOutput
+
 newState :: Git.Repo -> AnnexState
 newState gitrepo = AnnexState
 	{ repo = gitrepo
 	, backends = []
 	, remotes = []
 	, repoqueue = empty
-	, quiet = False
+	, output = NormalOutput
 	, force = False
 	, fast = False
 	, branchstate = startBranchState
diff --git a/Command/Init.hs b/Command/Init.hs
index 2abe4c6615..6ba7df6829 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -7,8 +7,6 @@
 
 module Command.Init where
 
-import Control.Monad
-
 import Command
 import qualified Annex
 import UUID
diff --git a/Content.hs b/Content.hs
index 1c2475240b..e4bbee5282 100644
--- a/Content.hs
+++ b/Content.hs
@@ -23,11 +23,10 @@ module Content (
 	saveState
 ) where
 
-import System.IO.Error (try)
 import System.Directory
 import Control.Monad.State (liftIO)
 import System.Path
-import Control.Monad (when, filterM)
+import Control.Monad
 import System.Posix.Files
 import System.FilePath
 import Data.Maybe
diff --git a/Messages.hs b/Messages.hs
index 36f0b89c5c..b2c871ede1 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -9,7 +9,6 @@ module Messages where
 
 import Control.Monad.State (liftIO)
 import System.IO
-import Control.Monad (unless)
 import Data.String.Utils
 
 import Types
@@ -17,8 +16,10 @@ import qualified Annex
 
 verbose :: Annex () -> Annex ()
 verbose a = do
-	q <- Annex.getState Annex.quiet
-	unless q a
+	output <- Annex.getState Annex.output
+	case output of
+		Annex.NormalOutput -> a
+		_ -> return ()
 
 showStart :: String -> String -> Annex ()
 showStart command file = verbose $ liftIO $ do
diff --git a/Options.hs b/Options.hs
index 7f78f44f62..768a1c289d 100644
--- a/Options.hs
+++ b/Options.hs
@@ -26,9 +26,9 @@ commonOptions =
 		"allow actions that may lose annexed data"
 	, Option ['F'] ["fast"] (NoArg (setfast True))
 		"avoid slow operations"
-	, Option ['q'] ["quiet"] (NoArg (setquiet True))
+	, Option ['q'] ["quiet"] (NoArg (setoutput Annex.QuietOutput))
 		"avoid verbose output"
-	, Option ['v'] ["verbose"] (NoArg (setquiet False))
+	, Option ['v'] ["verbose"] (NoArg (setoutput Annex.NormalOutput))
 		"allow verbose output (default)"
 	, Option ['d'] ["debug"] (NoArg (setdebug))
 		"show debug messages"
@@ -38,7 +38,7 @@ commonOptions =
 	where
 		setforce v = Annex.changeState $ \s -> s { Annex.force = v }
 		setfast v = Annex.changeState $ \s -> s { Annex.fast = v }
-		setquiet v = Annex.changeState $ \s -> s { Annex.quiet = v }
+		setoutput v = Annex.changeState $ \s -> s { Annex.output = v }
 		setforcebackend v = Annex.changeState $ \s -> s { Annex.forcebackend = Just v }
 		setdebug = liftIO $ updateGlobalLogger rootLoggerName $
 			setLevel DEBUG

From 2f4d4d1c4552a93a5f26a8a0a713e3916612329e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 1 Sep 2011 15:16:31 -0400
Subject: [PATCH 094/111] basic json support

This includes a generic JSONStream library built on top of Text.JSON
(somewhat hackishly).

It would be possible to stream out a single json document describing
all actions, but it's probably better for consumers if they can expect
one json document per line, so I did it that way instead.

Output from external programs used for transferring files is not
currently hidden when outputting json, which probably makes it not very
useful there. This may be dealt with if there is demand for json
output for --get or --move to be parsable.

The version, status, and find subcommands have hand-crafted output and
don't do json. The whereis subcommand needs to be modified to produce
useful json.
---
 Annex.hs              |  2 +-
 Messages.hs           | 61 +++++++++++++++++++++++++++++--------------
 Messages/JSON.hs      | 23 ++++++++++++++++
 Options.hs            |  2 ++
 Utility/JSONStream.hs | 44 +++++++++++++++++++++++++++++++
 debian/changelog      |  1 +
 debian/control        |  1 +
 doc/git-annex.mdwn    | 10 +++++--
 doc/install.mdwn      |  1 +
 9 files changed, 123 insertions(+), 22 deletions(-)
 create mode 100644 Messages/JSON.hs
 create mode 100644 Utility/JSONStream.hs

diff --git a/Annex.hs b/Annex.hs
index fac5d27e49..f5c3e4de45 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -64,7 +64,7 @@ data AnnexState = AnnexState
 	, cipher :: Maybe Cipher
 	}
 
-data OutputType = NormalOutput | QuietOutput
+data OutputType = NormalOutput | QuietOutput | JSONOutput
 
 newState :: Git.Repo -> AnnexState
 newState gitrepo = AnnexState
diff --git a/Messages.hs b/Messages.hs
index b2c871ede1..87d414f172 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -5,7 +5,22 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Messages where
+module Messages (
+	showStart,
+	showNote,
+	showAction,
+	showProgress,
+	showSideAction,
+	showOutput,
+	showLongNote,
+	showEndOk,
+	showEndFail,
+	showEndResult,
+	showErr,
+	warning,
+	indent,
+	setupConsole
+) where
 
 import Control.Monad.State (liftIO)
 import System.IO
@@ -13,21 +28,15 @@ import Data.String.Utils
 
 import Types
 import qualified Annex
-
-verbose :: Annex () -> Annex ()
-verbose a = do
-	output <- Annex.getState Annex.output
-	case output of
-		Annex.NormalOutput -> a
-		_ -> return ()
+import qualified Messages.JSON as JSON
 
 showStart :: String -> String -> Annex ()
-showStart command file = verbose $ liftIO $ do
+showStart command file = handle (JSON.start command file) $ do
 	putStr $ command ++ " " ++ file ++ " "
 	hFlush stdout
 
 showNote :: String -> Annex ()
-showNote s = verbose $ liftIO $ do
+showNote s = handle (JSON.note s) $ do
 	putStr $ "(" ++ s ++ ") "
 	hFlush stdout
 
@@ -35,28 +44,31 @@ showAction :: String -> Annex ()
 showAction s = showNote $ s ++ "..."
 
 showProgress :: Annex ()
-showProgress = verbose $ liftIO $ do
+showProgress = handle q $ do
 	putStr "."
 	hFlush stdout
 
 showSideAction :: String -> Annex ()
-showSideAction s = verbose $ liftIO $ putStrLn $ "(" ++ s ++ "...)"
+showSideAction s = handle q $ putStrLn $ "(" ++ s ++ "...)"
 
 showOutput :: Annex ()
-showOutput = verbose $ liftIO $ putStr "\n"
+showOutput = handle q $ putStr "\n"
 
 showLongNote :: String -> Annex ()
-showLongNote s = verbose $ liftIO $ putStr $ '\n' : indent s
+showLongNote s = handle (JSON.note s) $ putStr $ '\n' : indent s
 
 showEndOk :: Annex ()
-showEndOk = verbose $ liftIO $ putStrLn "ok"
+showEndOk = showEndResult True
 
 showEndFail :: Annex ()
-showEndFail = verbose $ liftIO $ putStrLn "failed"
+showEndFail = showEndResult False
 
 showEndResult :: Bool -> Annex ()
-showEndResult True = showEndOk
-showEndResult False = showEndFail
+showEndResult b = handle (JSON.end b) $ putStrLn msg
+	where
+		msg
+			| b = "ok"
+			| otherwise = "failed"
 
 showErr :: (Show a) => a -> Annex ()
 showErr e = liftIO $ do
@@ -65,7 +77,7 @@ showErr e = liftIO $ do
 
 warning :: String -> Annex ()
 warning w = do
-	verbose $ liftIO $ putStr "\n"
+	handle q $ putStr "\n"
 	liftIO $ do
 		hFlush stdout
 		hPutStrLn stderr $ indent w
@@ -85,3 +97,14 @@ setupConsole :: IO ()
 setupConsole = do
 	hSetBinaryMode stdout True
 	hSetBinaryMode stderr True
+
+handle :: IO () -> IO () -> Annex ()
+handle json normal = do
+	output <- Annex.getState Annex.output
+	case output of
+		Annex.NormalOutput -> liftIO normal
+		Annex.QuietOutput -> q
+		Annex.JSONOutput -> liftIO json
+
+q :: Monad m => m ()
+q = return ()
diff --git a/Messages/JSON.hs b/Messages/JSON.hs
new file mode 100644
index 0000000000..ee6ea34a32
--- /dev/null
+++ b/Messages/JSON.hs
@@ -0,0 +1,23 @@
+{- git-annex JSON output
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Messages.JSON (
+	start,
+	end,
+	note
+) where
+
+import qualified Utility.JSONStream as Stream
+
+start :: String -> String -> IO ()
+start command file = putStr $ Stream.start [("command", command), ("file", file)]
+
+end :: Bool -> IO ()
+end b = putStr $ Stream.add [("success", b)] ++ Stream.end
+
+note :: String -> IO ()
+note s = putStr $ Stream.add [("note", s)]
diff --git a/Options.hs b/Options.hs
index 768a1c289d..e0ca48c01b 100644
--- a/Options.hs
+++ b/Options.hs
@@ -30,6 +30,8 @@ commonOptions =
 		"avoid verbose output"
 	, Option ['v'] ["verbose"] (NoArg (setoutput Annex.NormalOutput))
 		"allow verbose output (default)"
+	, Option ['j'] ["json"] (NoArg (setoutput Annex.JSONOutput))
+		"enable JSON output"
 	, Option ['d'] ["debug"] (NoArg (setdebug))
 		"show debug messages"
 	, Option ['b'] ["backend"] (ReqArg setforcebackend paramName)
diff --git a/Utility/JSONStream.hs b/Utility/JSONStream.hs
new file mode 100644
index 0000000000..af3766948f
--- /dev/null
+++ b/Utility/JSONStream.hs
@@ -0,0 +1,44 @@
+{- Streaming JSON output.
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.JSONStream (
+	start,
+	add,
+	end
+) where
+
+import Text.JSON
+
+{- Text.JSON does not support building up a larger JSON document piece by
+   piece as a stream. To support streaming, a hack. The JSObject is converted
+   to a string with its final "}" is left off, allowing it to be added to
+   later. -}
+start :: JSON a => [(String, a)] -> String
+start l
+	| last s == endchar = take (length s - 1)  s
+	| otherwise = bad s
+	where
+		s = encodeStrict $ toJSObject l
+
+add :: JSON a => [(String, a)] -> String
+add l
+	| head s == startchar = ',' : drop 1 s
+	| otherwise = bad s
+	where
+		s = start l
+
+end :: String
+end = [endchar, '\n']
+
+startchar :: Char
+startchar = '{'
+
+endchar :: Char
+endchar = '}'
+
+bad :: String -> a
+bad s = error $ "Text.JSON returned unexpected string: " ++ s
diff --git a/debian/changelog b/debian/changelog
index 4ee0b80f2e..ca23ab4735 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,7 @@ git-annex (3.20110820) UNRELEASED; urgency=low
   * init: Make description an optional parameter.
   * unused, status: Sped up by avoiding unnecessary stats of annexed files.
   * unused --remote: Reduced memory use to 1/4th what was used before.
+  * Add --json switch, to produce machine-consumable output.
 
  -- Joey Hess   Tue, 23 Aug 2011 13:41:01 -0400
 
diff --git a/debian/control b/debian/control
index 63488dc68d..cb5a8212ab 100644
--- a/debian/control
+++ b/debian/control
@@ -14,6 +14,7 @@ Build-Depends:
 	libghc-hs3-dev (>= 0.5.6),
 	libghc-testpack-dev [any-i386 any-amd64],
 	libghc-monad-control-dev,
+	libghc-json-dev,
 	ikiwiki,
 	perlmagick,
 	git | git-core,
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index e7ac9adf7a..0a484a3842 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -337,12 +337,18 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * --quiet
 
-  Avoid the default verbose logging of what is done; only show errors
+  Avoid the default verbose display of what is done; only show errors
   and progress displays.
 
 * --verbose
 
-  Enable verbose logging.
+  Enable verbose display.
+
+* --json
+
+  Rather than the normal output, generate JSON. This is intended to be
+  parsed by programs that use git-annex. Each line of output is a JSON
+  object.
 
 * --debug
 
diff --git a/doc/install.mdwn b/doc/install.mdwn
index ac521da187..cd51b96d23 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -28,6 +28,7 @@ To build and use git-annex, you will need:
   * [QuickCheck 2](http://hackage.haskell.org/package/QuickCheck)
   * [HTTP](http://hackage.haskell.org/package/HTTP)
   * [hS3](http://hackage.haskell.org/package/hS3) (optional, but recommended)
+  * [json](http://hackage.haskell.org/package/json)
 * Shell commands
   * [git](http://git-scm.com/)
   * [uuid](http://www.ossp.org/pkg/lib/uuid/)

From 5bc32c7f3438a329878f4da7ad0b514c12a54332 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 1 Sep 2011 16:02:01 -0400
Subject: [PATCH 095/111] add json formatted list of remotes

Wherever a list of remotes is shown, --json now enables a json formatted
list.
---
 Command/Fsck.hs    |  2 +-
 Command/Whereis.hs |  2 +-
 Remote.hs          | 35 +++++++++++++++++++++++++----------
 3 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 6ccec05fb3..bad60d30dd 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -131,7 +131,7 @@ checkKeyNumCopies key file numcopies = do
 	let present = length safelocations
 	if present < needed
 		then do
-			ppuuids <- Remote.prettyPrintUUIDs untrustedlocations
+			ppuuids <- Remote.prettyPrintUUIDs "untrusted" untrustedlocations
 			warning $ missingNote (filename file key) present needed ppuuids
 			return False
 		else return True
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 314fef7826..f80c823b70 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -33,7 +33,7 @@ perform key = do
 	if null uuids
 		then stop
 		else do
-			pp <- prettyPrintUUIDs uuids
+			pp <- prettyPrintUUIDs "whereis" uuids
 			showLongNote pp
 			showOutput
 			next $ return True
diff --git a/Remote.hs b/Remote.hs
index 2c883f1a8f..ad1768da64 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -30,11 +30,14 @@ module Remote (
 ) where
 
 import Control.Monad (filterM)
+import Control.Monad.State (liftIO)
 import Data.List
 import qualified Data.Map as M
 import Data.String.Utils
 import Data.Maybe
 import Control.Applicative
+import Text.JSON
+import Text.JSON.Generic
 
 import Types
 import Types.Remote
@@ -44,6 +47,7 @@ import Config
 import Trust
 import LocationLog
 import Messages
+import qualified Utility.JSONStream
 import RemoteLog
 
 import qualified Remote.Git
@@ -119,23 +123,34 @@ nameToUUID n = do
 		invertMap = M.fromList . map swap . M.toList
 		swap (a, b) = (b, a)
 
-{- Pretty-prints a list of UUIDs of remotes. -}
-prettyPrintUUIDs :: [UUID] -> Annex String
-prettyPrintUUIDs uuids = do
+{- Pretty-prints a list of UUIDs of remotes, for human display.
+ -
+ - Shows descriptions from the uuid log, falling back to remote names,
+ - as some remotes may not be in the uuid log.
+ -
+ - When JSON is enabled, also generates a machine-readable description
+ - of the UUIDs. -}
+prettyPrintUUIDs :: String -> [UUID] -> Annex String
+prettyPrintUUIDs desc uuids = do
 	here <- getUUID =<< Annex.gitRepo
-	-- Show descriptions from the uuid log, falling back to remote names,
-	-- as some remotes may not be in the uuid log
 	m <- M.union <$> uuidMap <*> availMap
-	return $ unwords $ map (\u -> "\t" ++ prettify m u here ++ "\n") uuids
+	liftIO . putStr $ Utility.JSONStream.add
+		[(desc, map (jsonify m here) uuids)]
+	return $ unwords $ map (\u -> "\t" ++ prettify m here u ++ "\n") uuids
 	where
 		availMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList
-		prettify m u here = base ++ ishere
+		findlog m u = M.findWithDefault "" u m
+		prettify m here u = base ++ ishere
 			where
 				base = if not $ null $ findlog m u
 					then u ++ "  -- " ++ findlog m u
 					else u
 				ishere = if here == u then " <-- here" else ""
-		findlog m u = M.findWithDefault "" u m
+		jsonify m here u = toJSObject
+			[ ("uuid", toJSON u)
+			, ("description", toJSON $ findlog m u)
+			, ("here", toJSON $ here == u)
+			]
 
 {- Filters a list of remotes to ones that have the listed uuids. -}
 remotesWithUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
@@ -186,8 +201,8 @@ showLocations key exclude = do
 	untrusteduuids <- trustGet UnTrusted
 	let uuidswanted = filteruuids uuids (u:exclude++untrusteduuids) 
 	let uuidsskipped = filteruuids uuids (u:exclude++uuidswanted)
-	ppuuidswanted <- Remote.prettyPrintUUIDs uuidswanted
-	ppuuidsskipped <- Remote.prettyPrintUUIDs uuidsskipped
+	ppuuidswanted <- Remote.prettyPrintUUIDs "wanted" uuidswanted
+	ppuuidsskipped <- Remote.prettyPrintUUIDs "skipped" uuidsskipped
 	showLongNote $ message ppuuidswanted ppuuidsskipped
 	where
 		filteruuids l x = filter (`notElem` x) l

From e4a74c0dc533922ce1e7cd56dfa7ac08efc064cb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 1 Sep 2011 16:11:21 -0400
Subject: [PATCH 096/111] close bug with some caveats

---
 doc/bugs/add_script-friendly_output_options.mdwn | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/doc/bugs/add_script-friendly_output_options.mdwn b/doc/bugs/add_script-friendly_output_options.mdwn
index efcd54553f..08f7e6c7b8 100644
--- a/doc/bugs/add_script-friendly_output_options.mdwn
+++ b/doc/bugs/add_script-friendly_output_options.mdwn
@@ -1,3 +1,18 @@
 I have a need to use git-annex from a larger program. It'd be great if the information output by some of the commands that is descriptive (for example, whereis) could be sent to stdout in a machine-readable format like (preferably) JSON, or XML. That way I can simply read in the output of the command and use the data directly instead of having to parse it via regexes or other such string manipulation.
 
 This could perhaps be triggered by a --json or --xml flag to the relevant commands.
+
+> This is [[done]], --json is supported by all commands, more or less.
+> 
+> Caveats: 
+> 
+>  * the version, status, and find commands produce custom output and so
+>    no json. This could change for version and status; find needs to just
+>    be a simple list of files, I think 
+>  * The "note" fields may repeat multiple times per object with different
+>    notes and are of course not machine readable, and subject to change.
+>  * Output of helper commands like rsync is not diverted away, and
+>    could clutter up the json output badly. Should only affect commands
+>    that transfer data.
+> 
+> --[[Joey]] 

From cb5dacfd403a5ca4e3bdeb3eb3d0009136966f91 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 1 Sep 2011 16:15:57 -0400
Subject: [PATCH 097/111] rsync and wget use stderr for progress, so no problem

---
 doc/bugs/add_script-friendly_output_options.mdwn | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/add_script-friendly_output_options.mdwn b/doc/bugs/add_script-friendly_output_options.mdwn
index 08f7e6c7b8..7d7bdfc51a 100644
--- a/doc/bugs/add_script-friendly_output_options.mdwn
+++ b/doc/bugs/add_script-friendly_output_options.mdwn
@@ -13,6 +13,7 @@ This could perhaps be triggered by a --json or --xml flag to the relevant comman
 >    notes and are of course not machine readable, and subject to change.
 >  * Output of helper commands like rsync is not diverted away, and
 >    could clutter up the json output badly. Should only affect commands
->    that transfer data.
+>    that transfer data. And AFAICS, wget and rsync both output their
+>    progress displays to stderr, so shouldn't be a problem.
 > 
 > --[[Joey]] 

From d1dd40c49bb95416adec5b33e4dc1e7672a2f7f7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Sep 2011 16:44:04 -0400
Subject: [PATCH 098/111] avoid showing json lists of remotes when not in json
 mode

---
 Messages.hs      |  6 ++++++
 Messages/JSON.hs | 10 ++++++++--
 Remote.hs        |  5 +----
 3 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/Messages.hs b/Messages.hs
index 87d414f172..faa4dbcdec 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -19,12 +19,14 @@ module Messages (
 	showErr,
 	warning,
 	indent,
+	maybeShowJSON,
 	setupConsole
 ) where
 
 import Control.Monad.State (liftIO)
 import System.IO
 import Data.String.Utils
+import Text.JSON
 
 import Types
 import qualified Annex
@@ -106,5 +108,9 @@ handle json normal = do
 		Annex.QuietOutput -> q
 		Annex.JSONOutput -> liftIO json
 
+{- Shows a JSON value only when in json mode. -}
+maybeShowJSON :: JSON a => [(String, a)] -> Annex ()
+maybeShowJSON v = handle (JSON.add v) q
+
 q :: Monad m => m ()
 q = return ()
diff --git a/Messages/JSON.hs b/Messages/JSON.hs
index ee6ea34a32..fb95f550e8 100644
--- a/Messages/JSON.hs
+++ b/Messages/JSON.hs
@@ -8,9 +8,12 @@
 module Messages.JSON (
 	start,
 	end,
-	note
+	note,
+	add
 ) where
 
+import Text.JSON
+
 import qualified Utility.JSONStream as Stream
 
 start :: String -> String -> IO ()
@@ -20,4 +23,7 @@ end :: Bool -> IO ()
 end b = putStr $ Stream.add [("success", b)] ++ Stream.end
 
 note :: String -> IO ()
-note s = putStr $ Stream.add [("note", s)]
+note s = add [("note", s)]
+
+add :: JSON a => [(String, a)] -> IO ()
+add v = putStr $ Stream.add v
diff --git a/Remote.hs b/Remote.hs
index ad1768da64..e54d2e2334 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -30,7 +30,6 @@ module Remote (
 ) where
 
 import Control.Monad (filterM)
-import Control.Monad.State (liftIO)
 import Data.List
 import qualified Data.Map as M
 import Data.String.Utils
@@ -47,7 +46,6 @@ import Config
 import Trust
 import LocationLog
 import Messages
-import qualified Utility.JSONStream
 import RemoteLog
 
 import qualified Remote.Git
@@ -134,8 +132,7 @@ prettyPrintUUIDs :: String -> [UUID] -> Annex String
 prettyPrintUUIDs desc uuids = do
 	here <- getUUID =<< Annex.gitRepo
 	m <- M.union <$> uuidMap <*> availMap
-	liftIO . putStr $ Utility.JSONStream.add
-		[(desc, map (jsonify m here) uuids)]
+	maybeShowJSON [(desc, map (jsonify m here) uuids)]
 	return $ unwords $ map (\u -> "\t" ++ prettify m here u ++ "\n") uuids
 	where
 		availMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList

From d238bbd9d9f5e96334b9214c7aeca7875054e99e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Sep 2011 21:32:05 -0400
Subject: [PATCH 099/111] releasing version 3.20110902

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index ca23ab4735..35144f678d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20110820) UNRELEASED; urgency=low
+git-annex (3.20110902) unstable; urgency=low
 
   * Set EMAIL when running test suite so that git does not need to be
     configured first. Closes: #638998
@@ -8,7 +8,7 @@ git-annex (3.20110820) UNRELEASED; urgency=low
   * unused --remote: Reduced memory use to 1/4th what was used before.
   * Add --json switch, to produce machine-consumable output.
 
- -- Joey Hess   Tue, 23 Aug 2011 13:41:01 -0400
+ -- Joey Hess   Fri, 02 Sep 2011 21:20:37 -0400
 
 git-annex (3.20110819) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index a37c187c90..7850f88a77 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110819
+Version: 3.20110902
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From ea621d0c2d418c6dfe247b0aa3f78976c002b2b2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Sep 2011 21:32:21 -0400
Subject: [PATCH 100/111] add news item for git-annex 3.20110902

---
 doc/news/version_3.20110705.mdwn | 9 ---------
 doc/news/version_3.20110902.mdwn | 9 +++++++++
 2 files changed, 9 insertions(+), 9 deletions(-)
 delete mode 100644 doc/news/version_3.20110705.mdwn
 create mode 100644 doc/news/version_3.20110902.mdwn

diff --git a/doc/news/version_3.20110705.mdwn b/doc/news/version_3.20110705.mdwn
deleted file mode 100644
index bb4665c047..0000000000
--- a/doc/news/version_3.20110705.mdwn
+++ /dev/null
@@ -1,9 +0,0 @@
-git-annex 3.20110705 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * uninit: Delete the git-annex branch and .git/annex/
-   * unannex: In --fast mode, file content is left in the annex, and a
-     hard link made to it.
-   * uninit: Use unannex in --fast mode, to support unannexing multiple
-     files that link to the same content.
-   * Drop the dependency on the haskell curl bindings, use regular haskell HTTP.
-   * Fix a pipeline stall when upgrading (caused by #624389)."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20110902.mdwn b/doc/news/version_3.20110902.mdwn
new file mode 100644
index 0000000000..e354874ea3
--- /dev/null
+++ b/doc/news/version_3.20110902.mdwn
@@ -0,0 +1,9 @@
+git-annex 3.20110902 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Set EMAIL when running test suite so that git does not need to be
+     configured first. Closes: #[638998](http://bugs.debian.org/638998)
+   * The wget command will now be used in preference to curl, if available.
+   * init: Make description an optional parameter.
+   * unused, status: Sped up by avoiding unnecessary stats of annexed files.
+   * unused --remote: Reduced memory use to 1/4th what was used before.
+   * Add --json switch, to produce machine-consumable output."""]]
\ No newline at end of file

From c5c525d930480ccbf230420efb3b07cfd3ea78d7 Mon Sep 17 00:00:00 2001
From: DavidEdmondson 
Date: Mon, 5 Sep 2011 15:43:26 +0000
Subject: [PATCH 101/111] Added a comment: Is it necessary to commit after the
 'drop'?

---
 .../comment_1_cb65e7c510b75be1c51f655b058667c6._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/walkthrough/removing_files/comment_1_cb65e7c510b75be1c51f655b058667c6._comment

diff --git a/doc/walkthrough/removing_files/comment_1_cb65e7c510b75be1c51f655b058667c6._comment b/doc/walkthrough/removing_files/comment_1_cb65e7c510b75be1c51f655b058667c6._comment
new file mode 100644
index 0000000000..1c8719cecd
--- /dev/null
+++ b/doc/walkthrough/removing_files/comment_1_cb65e7c510b75be1c51f655b058667c6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="DavidEdmondson"
+ subject="Is it necessary to commit after the 'drop'?"
+ date="2011-09-05T15:43:25Z"
+ content="""
+In fact is it possible? Nothing changed as far as git is concerned.
+
+"""]]

From f0777d9b5a90326c495d557f79c8420e0c7d284a Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 5 Sep 2011 15:59:27 +0000
Subject: [PATCH 102/111] Added a comment

---
 .../comment_2_64709ea4558915edd5c8ca4486965b07._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/walkthrough/removing_files/comment_2_64709ea4558915edd5c8ca4486965b07._comment

diff --git a/doc/walkthrough/removing_files/comment_2_64709ea4558915edd5c8ca4486965b07._comment b/doc/walkthrough/removing_files/comment_2_64709ea4558915edd5c8ca4486965b07._comment
new file mode 100644
index 0000000000..f5fb8dc7f5
--- /dev/null
+++ b/doc/walkthrough/removing_files/comment_2_64709ea4558915edd5c8ca4486965b07._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-09-05T15:59:27Z"
+ content="""
+Good catch. It used to be necessary before there was a git-annex branch, but not now.
+"""]]

From dbef6a045c3f80510af3dd5ce80d94371a21a229 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 5 Sep 2011 11:56:30 -0400
Subject: [PATCH 103/111] remove now-unnecessary commit after drop

---
 doc/walkthrough/removing_files.mdwn | 1 -
 1 file changed, 1 deletion(-)

diff --git a/doc/walkthrough/removing_files.mdwn b/doc/walkthrough/removing_files.mdwn
index 85a7d50a6b..0df6e82a1f 100644
--- a/doc/walkthrough/removing_files.mdwn
+++ b/doc/walkthrough/removing_files.mdwn
@@ -3,4 +3,3 @@ has the file before removing it.
 
 	# git annex drop iso/debian.iso
 	drop iso/Debian_5.0.iso ok
-	# git commit -a -m "freed up space"

From 14f75ced75aa73d4f8dbc45cdc697bac1535fb37 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 5 Sep 2011 11:57:58 -0400
Subject: [PATCH 104/111] remove now unnecessary commit after get

---
 doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn | 1 -
 1 file changed, 1 deletion(-)

diff --git a/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn b/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn
index 936d088f1f..cfb70aaf9a 100644
--- a/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn
+++ b/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn
@@ -15,4 +15,3 @@ it:
 	# sudo mount /media/usb
 	# git annex get video/hackity_hack_and_kaxxt.mov
 	get video/hackity_hack_and_kaxxt.mov (from usbdrive...) ok
-	# git commit -a -m "got a video I want to rewatch on the plane"

From 07125dca53ece4896a83fad00b7f43101855ce9c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Sep 2011 13:46:08 -0400
Subject: [PATCH 105/111] Improve display of newlines around error and warning
 messages.

---
 CmdLine.hs                |  2 +-
 Command.hs                |  1 -
 Messages.hs               | 11 ++++++-----
 debian/changelog          |  6 ++++++
 doc/bugs/fsck_output.mdwn | 10 ++++++++++
 5 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 2652e5b8fc..4ccd2c2c2f 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -89,8 +89,8 @@ tryRun' errnum state (a:as) = do
 	case result of
 		Left err -> do
 			Annex.eval state $ do
-				showEndFail
 				showErr err
+				showEndFail
 			tryRun' (errnum + 1) state as
 		Right (True,state') -> tryRun' errnum state' as
 		Right (False,state') -> tryRun' (errnum + 1) state' as
diff --git a/Command.hs b/Command.hs
index 21c50f9c02..78f9823fb3 100644
--- a/Command.hs
+++ b/Command.hs
@@ -103,7 +103,6 @@ doCommand = start
 		stage a b = b >>= a
 		success = return True
 		failure = do
-			showOutput -- avoid clutter around error message
 			showEndFail
 			return False
 
diff --git a/Messages.hs b/Messages.hs
index faa4dbcdec..4922519819 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -73,16 +73,17 @@ showEndResult b = handle (JSON.end b) $ putStrLn msg
 			| otherwise = "failed"
 
 showErr :: (Show a) => a -> Annex ()
-showErr e = liftIO $ do
-	hFlush stdout
-	hPutStrLn stderr $ "git-annex: " ++ show e
+showErr e = warning' $ "git-annex: " ++ show e
 
 warning :: String -> Annex ()
-warning w = do
+warning w = warning' (indent w)
+
+warning' :: String -> Annex ()
+warning' w = do
 	handle q $ putStr "\n"
 	liftIO $ do
 		hFlush stdout
-		hPutStrLn stderr $ indent w
+		hPutStrLn stderr w
 
 indent :: String -> String
 indent s = join "\n" $ map (\l -> "  " ++ l) $ lines s
diff --git a/debian/changelog b/debian/changelog
index 35144f678d..9ab3b4fd3d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20110903) UNRELEASED; urgency=low
+
+  * Improve display of newlines around error and warning messages.
+
+ -- Joey Hess   Tue, 06 Sep 2011 13:45:16 -0400
+
 git-annex (3.20110902) unstable; urgency=low
 
   * Set EMAIL when running test suite so that git does not need to be
diff --git a/doc/bugs/fsck_output.mdwn b/doc/bugs/fsck_output.mdwn
index 90af1600d8..1b00dd7b37 100644
--- a/doc/bugs/fsck_output.mdwn
+++ b/doc/bugs/fsck_output.mdwn
@@ -34,3 +34,13 @@ The newline is in the wrong place and confuses the user. It should be printed _a
 > failed
 > 
 > --[[Joey]] 
+
+>> Well, I fixed this in all cases except a thrown non-IO error (last
+>> example aboce), which output is printed by haskell's runtime. I'd
+>> have to add a second error handler to handle those, and it's not
+>> clear what it would do. Often an error will occur before anything
+>> else is printed, and then the current behavior is right; if something
+>> has been printed it would be nice to have a newline before the error,
+>> but by the time the error is caught we'd be out of the annex monad
+>> and not really have any way to know if something has been printed.
+>> I think my fix is good enough [[done]] --[[Joey]] 

From fcfd2776cb010f35557a6e617d30327f4274e0ff Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl0-EtQjVUNysjom6sTlQxRUwkwD6uPx88"
 
Date: Tue, 6 Sep 2011 18:27:08 +0000
Subject: [PATCH 106/111]

---
 doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn

diff --git a/doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn b/doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn
new file mode 100644
index 0000000000..61c87b56a0
--- /dev/null
+++ b/doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn
@@ -0,0 +1,9 @@
+While following the instructions given at the OSX build page , I get this error:
+
+$ make
+ghc -O2 -Wall -ignore-package monads-fd -fspec-constr-count=5 --make git-annex
+
+Utility/JSONStream.hs:14:8:
+    Could not find module `Text.JSON':
+      Use -v to see a list of the files searched for.
+make: *** [git-annex] Error 1

From ca4eb842a4c7aa491c266e79ee1e1b340615ba01 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Sep 2011 14:36:21 -0400
Subject: [PATCH 107/111] add json to build scripts

---
 doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn | 2 ++
 doc/install/Fedora.mdwn                   | 1 +
 doc/install/OSX.mdwn                      | 1 +
 3 files changed, 4 insertions(+)

diff --git a/doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn b/doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn
index 61c87b56a0..43fb0323c4 100644
--- a/doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn
+++ b/doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn
@@ -7,3 +7,5 @@ Utility/JSONStream.hs:14:8:
     Could not find module `Text.JSON':
       Use -v to see a list of the files searched for.
 make: *** [git-annex] Error 1
+
+> Updated the instructions. [[done]] --[[Joey]] 
diff --git a/doc/install/Fedora.mdwn b/doc/install/Fedora.mdwn
index 0c1da2e6a5..17b8635218 100644
--- a/doc/install/Fedora.mdwn
+++ b/doc/install/Fedora.mdwn
@@ -11,6 +11,7 @@ sudo cabal install SHA
 sudo cabal install dataenc
 sudo cabal install monad-control
 sudo cabal install HTTP
+sudo cabal install json
 sudo cabal install hS3
 
 git clone git://git-annex.branchable.com/
diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index 983ac3a2fd..12502e0900 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -13,6 +13,7 @@ sudo cabal install SHA
 sudo cabal install dataenc
 sudo cabal install monad-control
 sudo cabal install HTTP
+sudo cabal install json
 sudo cabal install hS3 # optional
 
 # optional: this will enable the gnu tools, (to give sha224sum etc..., it does not override the BSD userland)

From 7722147e1625e0a16cfcf13fd967a4a961decaf7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Sep 2011 14:36:51 -0400
Subject: [PATCH 108/111] add josn build dep

---
 git-annex.cabal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index 7850f88a77..48ac48bb2d 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -31,7 +31,7 @@ Executable git-annex
   Build-Depends: haskell98, MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, HTTP,
-   base < 5, monad-control
+   base < 5, monad-control, json
 
 Executable git-annex-shell
   Main-Is: git-annex-shell.hs

From 3778e8897dcd8b2a5a54d076cc5d3a6d0a0bb689 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Sep 2011 14:48:15 -0400
Subject: [PATCH 109/111] switch to using cabal-install git-annex

---
 doc/install/Fedora.mdwn | 19 +------------------
 doc/install/OSX.mdwn    | 18 ++----------------
 2 files changed, 3 insertions(+), 34 deletions(-)

diff --git a/doc/install/Fedora.mdwn b/doc/install/Fedora.mdwn
index 17b8635218..068d5c111c 100644
--- a/doc/install/Fedora.mdwn
+++ b/doc/install/Fedora.mdwn
@@ -3,22 +3,5 @@ Installation recipe for Fedora 14.
 
 sudo yum install ghc cabal-install
 sudo cabal update
-sudo cabal install missingh
-sudo cabal install utf8-string
-sudo cabal install pcre-light
-sudo cabal install quickcheck
-sudo cabal install SHA
-sudo cabal install dataenc
-sudo cabal install monad-control
-sudo cabal install HTTP
-sudo cabal install json
-sudo cabal install hS3
-
-git clone git://git-annex.branchable.com/
-
-cd git-annex
-sudo make   # For some reason you need to use sudo here as otherwise the cabal installed packages doesn't seem to be there...
-sudo install git-annex
+sudo cabal install git-annex
 

-
-Originally posted by Jon at  --[[Joey]]
diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index 12502e0900..680c331ee6 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -4,26 +4,12 @@ Install Haskel Platform from [[http://hackage.haskell.org/platform/mac.html]]. T
 sudo port install git-core ossp-uuid md5sha1sum coreutils pcre
 
 sudo ln -s /opt/local/include/pcre.h  /usr/include/pcre.h # This is hack that allows pcre-light to find pcre
-sudo cabal update
-sudo cabal install missingh
-sudo cabal install utf8-string
-sudo cabal install pcre-light
-sudo cabal install quickcheck  
-sudo cabal install SHA
-sudo cabal install dataenc
-sudo cabal install monad-control
-sudo cabal install HTTP
-sudo cabal install json
-sudo cabal install hS3 # optional
 
 # optional: this will enable the gnu tools, (to give sha224sum etc..., it does not override the BSD userland)
 export PATH=$PATH:/opt/local/libexec/gnubin
 
-git clone  git://git-annex.branchable.com/
-
-cd git-annex
-make
-sudo make install
+sudo cabal update
+sudo cabal install git-annex
 

 
 Originally posted by Jon at  --[[Joey]], modified by [[kristianrumberg]]

From ebb92221fd2d42e360bf24a72b53971129908f01 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Sep 2011 15:35:13 -0400
Subject: [PATCH 110/111] Fix Makefile to work with cabal again.

---
 Makefile         | 4 +++-
 debian/changelog | 1 +
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 20d1347140..3f36928afb 100644
--- a/Makefile
+++ b/Makefile
@@ -29,7 +29,9 @@ Remote/S3.o: Remote/S3.hs
 		echo "** building without S3 support"; \
 	fi
 
-$(bins): $(sources)
+sources: $(sources)
+
+$(bins): sources
 	$(GHCMAKE) $@
 
 git-annex.1: doc/git-annex.mdwn
diff --git a/debian/changelog b/debian/changelog
index 9ab3b4fd3d..973623e200 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 git-annex (3.20110903) UNRELEASED; urgency=low
 
   * Improve display of newlines around error and warning messages.
+  * Fix Makefile to work with cabal again.
 
  -- Joey Hess   Tue, 06 Sep 2011 13:45:16 -0400
 

From 6fd0df7c2ff20710491923abfbb7a1bc4d3750bb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Sep 2011 15:54:21 -0400
Subject: [PATCH 111/111] releasing version 3.20110906

---
 debian/changelog | 2 +-
 git-annex.cabal  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 973623e200..0fca41ee5c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20110903) UNRELEASED; urgency=low
+git-annex (3.20110906) unstable; urgency=low
 
   * Improve display of newlines around error and warning messages.
   * Fix Makefile to work with cabal again.
diff --git a/git-annex.cabal b/git-annex.cabal
index 48ac48bb2d..34c8e76222 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110902
+Version: 3.20110906
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess