diff --git a/.gitignore b/.gitignore
index d5bf54c813..7d2504de6f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,7 +2,7 @@
 *.o
 test
 configure
-SysConfig.hs
+Build/SysConfig.hs
 git-annex
 git-annex-shell
 git-union-merge
@@ -13,7 +13,7 @@ doc/.ikiwiki
 html
 *.tix
 .hpc
-Touch.hs
-StatFS.hs
+Utility/Touch.hs
+Utility/StatFS.hs
 Remote/S3.hs
 dist
diff --git a/Annex.hs b/Annex.hs
index f7e3e29f82..f5c3e4de45 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -5,9 +5,12 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
+{-# LANGUAGE GeneralizedNewtypeDeriving #-}
+
 module Annex (
 	Annex,
 	AnnexState(..),
+	OutputType(..),
 	new,
 	run,
 	eval,
@@ -17,6 +20,8 @@ module Annex (
 ) where
 
 import Control.Monad.State
+import Control.Monad.IO.Control
+import Control.Applicative hiding (empty)
 
 import qualified Git
 import Git.Queue
@@ -28,7 +33,15 @@ import Types.TrustLevel
 import Types.UUID
 
 -- git-annex's monad
-type Annex = StateT AnnexState IO
+newtype Annex a = Annex { runAnnex :: StateT AnnexState IO a }
+	deriving (
+		Monad,
+		MonadIO,
+		MonadControlIO,
+		MonadState AnnexState,
+		Functor,
+		Applicative
+	)
 
 -- internal state storage
 data AnnexState = AnnexState
@@ -36,7 +49,7 @@ data AnnexState = AnnexState
 	, backends :: [Backend Annex]
 	, remotes :: [Remote Annex]
 	, repoqueue :: Queue
-	, quiet :: Bool
+	, output :: OutputType
 	, force :: Bool
 	, fast :: Bool
 	, branchstate :: BranchState
@@ -51,13 +64,15 @@ data AnnexState = AnnexState
 	, cipher :: Maybe Cipher
 	}
 
+data OutputType = NormalOutput | QuietOutput | JSONOutput
+
 newState :: Git.Repo -> AnnexState
 newState gitrepo = AnnexState
 	{ repo = gitrepo
 	, backends = []
 	, remotes = []
 	, repoqueue = empty
-	, quiet = False
+	, output = NormalOutput
 	, force = False
 	, fast = False
 	, branchstate = startBranchState
@@ -74,13 +89,13 @@ newState gitrepo = AnnexState
 
 {- Create and returns an Annex state object for the specified git repo. -}
 new :: Git.Repo -> IO AnnexState
-new gitrepo = newState `liftM` (liftIO . Git.configRead) gitrepo
+new gitrepo = newState <$> Git.configRead gitrepo
 
 {- performs an action in the Annex monad -}
 run :: AnnexState -> Annex a -> IO (a, AnnexState)
-run = flip runStateT
+run s a = runStateT (runAnnex a) s
 eval :: AnnexState -> Annex a -> IO a
-eval = flip evalStateT
+eval s a = evalStateT (runAnnex a) s
 
 {- Gets a value from the internal state, selected by the passed value
  - constructor. -}
diff --git a/AnnexQueue.hs b/AnnexQueue.hs
index 79116c48af..d155a7b81f 100644
--- a/AnnexQueue.hs
+++ b/AnnexQueue.hs
@@ -17,10 +17,9 @@ import Control.Monad (when, unless)
 import Annex
 import Messages
 import qualified Git.Queue
-import Utility
+import Utility.SafeCommand
 
-{- Adds a git command to the queue, possibly running previously queued
- - actions if enough have accumulated. -}
+{- Adds a git command to the queue. -}
 add :: String -> [CommandParam] -> [FilePath] -> Annex ()
 add command params files = do
 	q <- getState repoqueue
diff --git a/Backend.hs b/Backend.hs
index 3429e8f42c..75327de80b 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -32,9 +32,10 @@ import Messages
 -- When adding a new backend, import it here and add it to the list.
 import qualified Backend.WORM
 import qualified Backend.SHA
+import qualified Backend.URL
 
 list :: [Backend Annex]
-list = Backend.WORM.backends ++ Backend.SHA.backends
+list = Backend.WORM.backends ++ Backend.SHA.backends ++ Backend.URL.backends
 
 {- List of backends in the order to try them when storing a new key. -}
 orderedList :: Annex [Backend Annex]
@@ -65,7 +66,7 @@ orderedList = do
 genKey :: FilePath -> Maybe (Backend Annex) -> Annex (Maybe (Key, Backend Annex))
 genKey file trybackend = do
 	bs <- orderedList
-	let bs' = maybe bs (:bs) trybackend
+	let bs' = maybe bs (: bs) trybackend
 	genKey' bs' file
 genKey' :: [Backend Annex] -> FilePath -> Annex (Maybe (Key, Backend Annex))
 genKey' [] _ = return Nothing
@@ -121,8 +122,7 @@ lookupBackendName s = fromMaybe unknown $ maybeLookupBackendName s
 	where
 		unknown = error $ "unknown backend " ++ s
 maybeLookupBackendName :: String -> Maybe (Backend Annex)
-maybeLookupBackendName s =
-	if 1 /= length matches
-		then Nothing
-		else Just $ head matches
+maybeLookupBackendName s
+	| length matches == 1 = Just $ head matches
+	| otherwise = Nothing
 	where matches = filter (\b -> s == B.name b) list
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index c1d7136485..ed2a47db9b 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -23,8 +23,8 @@ import Content
 import Types
 import Types.Backend
 import Types.Key
-import Utility
-import qualified SysConfig
+import Utility.SafeCommand
+import qualified Build.SysConfig as SysConfig
 
 type SHASize = Int
 
@@ -32,7 +32,7 @@ sizes :: [Int]
 sizes = [1, 256, 512, 224, 384]
 
 backends :: [Backend Annex]
--- order is slightly significant; want sha1 first ,and more general
+-- order is slightly significant; want sha1 first, and more general
 -- sizes earlier
 backends = catMaybes $ map genBackend sizes ++ map genBackendE sizes
 
@@ -107,7 +107,7 @@ keyValueE size file = keyValue size file >>= maybe (return Nothing) addE
 				then "" -- probably not really an extension
 				else naiveextension
 
--- A key's checksum is checked during fsck.
+{- A key's checksum is checked during fsck. -}
 checkKeyChecksum :: SHASize -> Key -> Annex Bool
 checkKeyChecksum size key = do
 	g <- Annex.gitRepo
diff --git a/Backend/URL.hs b/Backend/URL.hs
new file mode 100644
index 0000000000..f20aa1f95e
--- /dev/null
+++ b/Backend/URL.hs
@@ -0,0 +1,28 @@
+{- git-annex "URL" backend -- keys whose content is available from urls.
+ -
+ - Copyright 2011 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Backend.URL (
+	backends,
+	fromUrl
+) where
+
+import Types.Backend
+import Types.Key
+import Types
+
+backends :: [Backend Annex]
+backends = [backend]
+
+backend :: Backend Annex
+backend = Types.Backend.Backend {
+	name = "URL",
+	getKey = const (return Nothing),
+	fsckKey = const (return True)
+}
+
+fromUrl :: String -> Key
+fromUrl url = stubKey { keyName = url, keyBackendName = "URL" }
diff --git a/Branch.hs b/Branch.hs
index 35e3050936..5008b2e200 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -14,11 +14,13 @@ module Branch (
 	files,
 	refExists,
 	hasOrigin,
+	hasSomeBranch,
 	name	
 ) where
 
 import Control.Monad (when, unless, liftM)
 import Control.Monad.State (liftIO)
+import Control.Applicative ((<$>))
 import System.FilePath
 import System.Directory
 import Data.String.Utils
@@ -36,6 +38,8 @@ import qualified Git
 import qualified Git.UnionMerge
 import qualified Annex
 import Utility
+import Utility.Conditional
+import Utility.SafeCommand
 import Types
 import Messages
 import Locations
@@ -124,7 +128,7 @@ getCache file = getState >>= handle
 
 {- Creates the branch, if it does not already exist. -}
 create :: Annex ()
-create = unlessM (refExists fullname) $ do
+create = unlessM hasBranch $ do
 	g <- Annex.gitRepo
 	e <- hasOrigin
 	if e
@@ -154,19 +158,14 @@ update = do
 		 -}
 		staged <- stageJournalFiles
 
+		refs <- siblingBranches
+		updated <- catMaybes <$> mapM updateRef refs
 		g <- Annex.gitRepo
-		r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
-		let refs = map (last . words) (lines r)
-		updated <- catMaybes `liftM` mapM updateRef refs
 		unless (null updated && not staged) $ liftIO $
 			Git.commit g "update" fullname (fullname:updated)
 		Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }
 		invalidateCache
 
-{- Does origin/git-annex exist? -}
-hasOrigin :: Annex Bool
-hasOrigin = refExists originname
-
 {- Checks if a git ref exists. -}
 refExists :: GitRef -> Annex Bool
 refExists ref = do
@@ -174,6 +173,26 @@ refExists ref = do
 	liftIO $ Git.runBool g "show-ref"
 		[Param "--verify", Param "-q", Param ref]
 
+{- Does the main git-annex branch exist? -}
+hasBranch :: Annex Bool
+hasBranch = refExists fullname
+
+{- Does origin/git-annex exist? -}
+hasOrigin :: Annex Bool
+hasOrigin = refExists originname
+
+{- Does the git-annex branch or a foo/git-annex branch exist? -}
+hasSomeBranch :: Annex Bool
+hasSomeBranch = not . null <$> siblingBranches
+
+{- List of all git-annex branches, including the main one and any
+ - from remotes. -}
+siblingBranches :: Annex [String]
+siblingBranches = do
+	g <- Annex.gitRepo
+	r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
+	return $ map (last . words) (lines r)
+
 {- Ensures that a given ref has been merged into the index. -}
 updateRef :: GitRef -> Annex (Maybe String)
 updateRef ref
@@ -305,7 +324,7 @@ getJournalFile file = do
 
 {- List of journal files. -}
 getJournalFiles :: Annex [FilePath]
-getJournalFiles = fmap (map fileJournal) getJournalFilesRaw
+getJournalFiles = map fileJournal <$> getJournalFilesRaw
 
 getJournalFilesRaw :: Annex [FilePath]
 getJournalFilesRaw = do
diff --git a/TestConfig.hs b/Build/TestConfig.hs
similarity index 94%
rename from TestConfig.hs
rename to Build/TestConfig.hs
index 8cfae7f0c7..e8a0d13368 100644
--- a/TestConfig.hs
+++ b/Build/TestConfig.hs
@@ -1,6 +1,6 @@
-{- Tests the system and generates SysConfig.hs. -}
+{- Tests the system and generates Build.SysConfig.hs. -}
 
-module TestConfig where
+module Build.TestConfig where
 
 import System.IO
 import System.Cmd
@@ -33,12 +33,12 @@ instance Show Config where
 			valuetype (MaybeStringConfig _) = "Maybe String"
 
 writeSysConfig :: [Config] -> IO ()
-writeSysConfig config = writeFile "SysConfig.hs" body
+writeSysConfig config = writeFile "Build/SysConfig.hs" body
 	where
 		body = unlines $ header ++ map show config ++ footer
 		header = [
 			  "{- Automatically generated. -}"
-			, "module SysConfig where"
+			, "module Build.SysConfig where"
 			, ""
 			]
 		footer = []
diff --git a/CmdLine.hs b/CmdLine.hs
index c33c497856..4ccd2c2c2f 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -22,10 +22,9 @@ import qualified Git
 import Content
 import Types
 import Command
-import Version
 import Options
 import Messages
-import UUID
+import Init
 
 {- Runs the passed command line. -}
 dispatch :: [String] -> [Command] -> [Option] -> String -> Git.Repo -> IO ()
@@ -45,7 +44,7 @@ parseCmd argv header cmds options = do
 		[] -> error $ "unknown command" ++ usagemsg
 		[command] -> do
 			_ <- sequence flags
-			when (cmdusesrepo command) checkVersion
+			checkCmdEnviron command
 			prepCommand command (drop 1 params)
 		_ -> error "internal error: multiple matching commands"
 	where
@@ -57,6 +56,10 @@ parseCmd argv header cmds options = do
 		lookupCmd cmd = filter (\c -> cmd  == cmdname c) cmds
 		usagemsg = "\n\n" ++ usage header cmds options
 
+{- Checks that the command can be run in the current environment. -}
+checkCmdEnviron :: Command -> Annex ()
+checkCmdEnviron command = when (cmdusesrepo command) ensureInitialized
+
 {- Usage message with lists of commands and options. -}
 usage :: String -> [Command] -> [Option] -> String
 usage header cmds options =
@@ -86,8 +89,8 @@ tryRun' errnum state (a:as) = do
 	case result of
 		Left err -> do
 			Annex.eval state $ do
-				showEndFail
 				showErr err
+				showEndFail
 			tryRun' (errnum + 1) state as
 		Right (True,state') -> tryRun' errnum state' as
 		Right (False,state') -> tryRun' (errnum + 1) state' as
@@ -95,9 +98,7 @@ tryRun' errnum _ [] = when (errnum > 0) $ error $ show errnum ++ " failed"
 
 {- Actions to perform each time ran. -}
 startup :: Annex Bool
-startup = do
-	prepUUID
-	return True
+startup = return True
 
 {- Cleanup actions. -}
 shutdown :: Annex Bool
diff --git a/Command.hs b/Command.hs
index 02bbd29d44..78f9823fb3 100644
--- a/Command.hs
+++ b/Command.hs
@@ -11,6 +11,7 @@ import Control.Monad.State (liftIO)
 import System.Directory
 import System.Posix.Files
 import Control.Monad (filterM, liftM, when)
+import Control.Applicative
 import System.Path.WildMatch
 import Text.Regex.PCRE.Light.Char8
 import Data.List
@@ -102,7 +103,6 @@ doCommand = start
 		stage a b = b >>= a
 		success = return True
 		failure = do
-			showOutput -- avoid clutter around error message
 			showEndFail
 			return False
 
@@ -178,14 +178,12 @@ withKeys :: CommandSeekKeys
 withKeys a params = return $ map (a . parse) params
 	where
 		parse p = fromMaybe (error "bad key") $ readKey p
-withTempFile :: CommandSeekStrings
-withTempFile a params = return $ map a params
 withNothing :: CommandSeekNothing
 withNothing a [] = return [a]
 withNothing _ _ = error "This command takes no parameters."
 
 backendPairs :: CommandSeekBackendFiles
-backendPairs a files = liftM (map a) $ Backend.chooseBackends files
+backendPairs a files = map a <$> Backend.chooseBackends files
 
 {- Filter out files those matching the exclude glob pattern,
  - if it was specified. -}
@@ -206,7 +204,7 @@ wildsRegex ws = compile regex []
 
 {- filter out symlinks -}	
 notSymlink :: FilePath -> IO Bool
-notSymlink f = liftM (not . isSymbolicLink) $ liftIO $ getSymbolicLinkStatus f
+notSymlink f = liftIO $ not . isSymbolicLink <$> getSymbolicLinkStatus f
 
 {- Descriptions of params used in usage messages. -}
 paramRepeating :: String -> String
@@ -273,4 +271,4 @@ preserveOrder orig new = collect orig new
  - of git file list commands, that assumption tends to hold.
  -}
 runPreserveOrder :: ([FilePath] -> IO [FilePath]) -> [FilePath] -> IO [FilePath]
-runPreserveOrder a files = liftM (preserveOrder files) (a files)
+runPreserveOrder a files = preserveOrder files <$> a files
diff --git a/Command/Add.hs b/Command/Add.hs
index c417b1ae84..587ce4c0ce 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -8,6 +8,7 @@
 module Command.Add where
 
 import Control.Monad.State (liftIO)
+import Control.Monad (when)
 import System.Posix.Files
 
 import Command
@@ -18,8 +19,10 @@ import LocationLog
 import Types
 import Content
 import Messages
-import Utility
-import Touch
+import Utility.Conditional
+import Utility.Touch
+import Utility.SafeCommand
+import Locations
 
 command :: [Command]
 command = [repoCommand "add" paramPath seek "add files to annex"]
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 1b12362e9f..55e51100ce 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -14,15 +14,16 @@ import System.Directory
 
 import Command
 import qualified Backend
+import qualified Utility.Url as Url
 import qualified Remote.Web
 import qualified Command.Add
 import qualified Annex
+import qualified Backend.URL
 import Messages
 import Content
 import PresenceLog
-import Types.Key
 import Locations
-import Utility
+import Utility.Path
 
 command :: [Command]
 command = [repoCommand "addurl" paramPath seek "add urls to annex"]
@@ -42,12 +43,17 @@ start s = do
 			
 perform :: String -> FilePath -> CommandPerform
 perform url file = do
+	fast <- Annex.getState Annex.fast
+	if fast then nodownload url file else download url file
+
+download :: String -> FilePath -> CommandPerform
+download url file = do
 	g <- Annex.gitRepo
 	showAction $ "downloading " ++ url ++ " "
-	let dummykey = stubKey { keyName = url, keyBackendName = "URL" }
+	let dummykey = Backend.URL.fromUrl url
 	let tmp = gitAnnexTmpLocation g dummykey
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
-	ok <- Remote.Web.download [url] tmp
+	ok <- Url.download url tmp
 	if ok
 		then do
 			[(_, backend)] <- Backend.chooseBackends [file]
@@ -57,9 +63,16 @@ perform url file = do
 				Just (key, _) -> do
 					moveAnnex key tmp
 					Remote.Web.setUrl key url InfoPresent
-					next $ Command.Add.cleanup file key
+					next $ Command.Add.cleanup file key True
 		else stop
 
+nodownload :: String -> FilePath -> CommandPerform
+nodownload url file = do
+	let key = Backend.URL.fromUrl url
+	Remote.Web.setUrl key url InfoPresent
+	
+	next $ Command.Add.cleanup file key False
+
 url2file :: URI -> IO FilePath
 url2file url = do
 	let parts = filter safe $ split "/" $ uriPath url
@@ -75,8 +88,7 @@ url2file url = do
 			e <- doesFileExist file
 			when e $ error "already have this url"
 			return file
-		safe s
-			| null s = False
-			| s == "." = False
-			| s == ".." = False
-			| otherwise = True
+		safe "" = False
+		safe "." = False
+		safe ".." = False
+		safe _ = True
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index d8dbff03af..1b1bb3c34b 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -14,7 +14,7 @@ import Command
 import UUID
 
 command :: [Command]
-command = [standaloneCommand "configlist" paramNothing seek
+command = [repoCommand "configlist" paramNothing seek
 		"outputs relevant git configuration"]
 
 seek :: [CommandSeek]
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 14f098349e..6e688d6632 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -15,6 +15,7 @@ import Types
 import Content
 import Messages
 import Utility
+import Utility.Conditional
 import Trust
 import Config
 
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 41bcd6aa78..4ad2aa85bb 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -22,7 +22,7 @@ import qualified Command.Move
 import qualified Remote
 import qualified Git
 import Types.Key
-import Utility
+import Utility.Conditional
 
 type UnusedMap = M.Map String Key
 
diff --git a/Command/Find.hs b/Command/Find.hs
index 9d760ff5a8..0716c5297e 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -11,7 +11,7 @@ import Control.Monad.State (liftIO)
 
 import Command
 import Content
-import Utility
+import Utility.Conditional
 
 command :: [Command]
 command = [repoCommand "find" (paramOptional $ paramRepeating paramPath) seek
diff --git a/Command/Fix.hs b/Command/Fix.hs
index 47b0c4c9a0..b24f8e33c2 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -13,7 +13,8 @@ import System.Directory
 
 import Command
 import qualified AnnexQueue
-import Utility
+import Utility.Path
+import Utility.SafeCommand
 import Content
 import Messages
 
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index d59f1de397..89c3f4e912 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -14,10 +14,11 @@ import Control.Monad (unless)
 
 import Command
 import qualified AnnexQueue
-import Utility
+import Utility.SafeCommand
 import Content
 import Messages
 import Types.Key
+import Utility.Path
 
 command :: [Command]
 command = [repoCommand "fromkey" paramPath seek
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 0d3ecb58f1..bad60d30dd 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -27,6 +27,7 @@ import LocationLog
 import Locations
 import Trust
 import Utility.DataUnits
+import Utility.Path
 import Config
 
 command :: [Command]
@@ -130,7 +131,7 @@ checkKeyNumCopies key file numcopies = do
 	let present = length safelocations
 	if present < needed
 		then do
-			ppuuids <- Remote.prettyPrintUUIDs untrustedlocations
+			ppuuids <- Remote.prettyPrintUUIDs "untrusted" untrustedlocations
 			warning $ missingNote (filename file key) present needed ppuuids
 			return False
 		else return True
diff --git a/Command/Init.hs b/Command/Init.hs
index 71e87050d8..6ba7df6829 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -7,31 +7,21 @@
 
 module Command.Init where
 
-import Control.Monad.State (liftIO)
-import Control.Monad (when, unless)
-import System.Directory
-
 import Command
 import qualified Annex
-import qualified Git
-import qualified Branch
 import UUID
-import Version
 import Messages
-import Types
-import Utility
+import Init
 	
 command :: [Command]
 command = [standaloneCommand "init" paramDesc seek
-		"initialize git-annex with repository description"]
+		"initialize git-annex"]
 
 seek :: [CommandSeek]
 seek = [withWords start]
 
 start :: CommandStartWords
 start ws = do
-	when (null description) $
-		error "please specify a description of this repository\n"
 	showStart "init" description
 	next $ perform description
 	where
@@ -39,34 +29,8 @@ start ws = do
 
 perform :: String -> CommandPerform
 perform description = do
-	Branch.create
+	initialize
 	g <- Annex.gitRepo
 	u <- getUUID g
-	setVersion
 	describeUUID u description
-	unless (Git.repoIsLocalBare g) $
-		gitPreCommitHookWrite g
 	next $ return True
-
-{- set up a git pre-commit hook, if one is not already present -}
-gitPreCommitHookWrite :: Git.Repo -> Annex ()
-gitPreCommitHookWrite repo = do
-	exists <- liftIO $ doesFileExist hook
-	if exists
-		then warning $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring"
-		else liftIO $ do
-			viaTmp writeFile hook preCommitScript
-			p <- getPermissions hook
-			setPermissions hook $ p {executable = True}
-	where
-		hook = preCommitHook repo
-
-preCommitHook :: Git.Repo -> FilePath
-preCommitHook repo = 
-	Git.workTree repo ++ "/" ++ Git.gitDir repo ++ "/hooks/pre-commit"
-
-preCommitScript :: String
-preCommitScript = 
-		"#!/bin/sh\n" ++
-		"# automatically configured by git-annex\n" ++ 
-		"git annex pre-commit .\n"
diff --git a/Command/Lock.hs b/Command/Lock.hs
index d39df5f335..77d1ff94f9 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -13,7 +13,7 @@ import System.Directory
 import Command
 import Messages
 import qualified AnnexQueue
-import Utility
+import Utility.SafeCommand
 	
 command :: [Command]
 command = [repoCommand "lock" paramPath seek "undo unlock command"]
diff --git a/Command/Map.hs b/Command/Map.hs
index 07f127f14e..ef8e04d909 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -19,10 +19,10 @@ import qualified Annex
 import qualified Git
 import Messages
 import Types
-import Utility
+import Utility.SafeCommand
 import UUID
 import Trust
-import Remote.Ssh
+import Utility.Ssh
 import qualified Utility.Dot as Dot
 
 -- a link from the first repository to the second (its remote)
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 495bf9fb63..6ad7e239c9 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -8,6 +8,7 @@
 module Command.Migrate where
 
 import Control.Monad.State (liftIO)
+import Control.Applicative
 import System.Posix.Files
 import System.Directory
 import System.FilePath
@@ -20,7 +21,7 @@ import Locations
 import Types
 import Content
 import Messages
-import Utility
+import Utility.Conditional
 import qualified Command.Add
 
 command :: [Command]
@@ -39,7 +40,7 @@ start (file, b) = isAnnexed file $ \(key, oldbackend) -> do
 			next $ perform file key newbackend
 		else stop
 	where
-		choosebackend Nothing = return . head =<< Backend.orderedList
+		choosebackend Nothing = head <$> Backend.orderedList
 		choosebackend (Just backend) = return backend
 
 {- Checks if a key is upgradable to a newer representation. -}
@@ -72,7 +73,7 @@ perform file oldkey newbackend = do
 				then do
 					-- Update symlink to use the new key.
 					liftIO $ removeFile file
-					next $ Command.Add.cleanup file newkey
+					next $ Command.Add.cleanup file newkey True
 				else stop
 	where
 		cleantmp t = whenM (doesFileExist t) $ removeFile t
diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index e2f7c74abb..be6163558f 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -13,8 +13,8 @@ import System.Exit
 import Command
 import CmdLine
 import Content
-import Utility
 import Utility.RsyncFile
+import Utility.Conditional
 
 command :: [Command]
 command = [repoCommand "recvkey" paramKey seek
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index 02fedb349e..f676ae947a 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -14,8 +14,8 @@ import Locations
 import qualified Annex
 import Command
 import Content
-import Utility
 import Utility.RsyncFile
+import Utility.Conditional
 import Messages
 
 command :: [Command]
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index f2a5259bac..2f6f9ea9ee 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -10,7 +10,7 @@ module Command.SetKey where
 import Control.Monad.State (liftIO)
 
 import Command
-import Utility
+import Utility.SafeCommand
 import LocationLog
 import Content
 import Messages
@@ -20,7 +20,7 @@ command = [repoCommand "setkey" paramPath seek
 	"sets annexed content for a key using a temp file"]
 
 seek :: [CommandSeek]
-seek = [withTempFile start]
+seek = [withStrings start]
 
 {- Sets cached content for a key. -}
 start :: CommandStartString
diff --git a/Command/Status.hs b/Command/Status.hs
index aef4df2329..5c82744b10 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -8,6 +8,7 @@
 module Command.Status where
 
 import Control.Monad.State
+import Control.Applicative
 import Data.Maybe
 import System.IO
 import Data.List
@@ -112,12 +113,10 @@ total_annex_size = stat "total annex size" $
 	cachedKeysReferenced >>= keySizeSum
 
 local_annex_keys :: Stat
-local_annex_keys = stat "local annex keys" $ 
-	return . show . snd =<< cachedKeysPresent
+local_annex_keys = stat "local annex keys" $ show . snd <$> cachedKeysPresent
 
 total_annex_keys :: Stat
-total_annex_keys = stat "total annex keys" $
-	return . show . snd =<< cachedKeysReferenced
+total_annex_keys = stat "total annex keys" $ show . snd <$> cachedKeysReferenced
 
 tmp_size :: Stat
 tmp_size = staleSize "temporary directory size" gitAnnexTmpDir
@@ -126,8 +125,7 @@ bad_data_size :: Stat
 bad_data_size = staleSize "bad keys size" gitAnnexBadDir
 
 backend_usage :: Stat
-backend_usage = stat "backend usage" $
-	return . usage =<< cachedKeysReferenced
+backend_usage = stat "backend usage" $ usage <$> cachedKeysReferenced
 	where
 		usage (ks, _) = pp "" $ sort $ map swap $ splits ks
 		splits :: [Key] -> [(String, Integer)]
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 960f99722f..54ef2fc68e 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -16,7 +16,8 @@ import Command
 import qualified Command.Drop
 import qualified Annex
 import qualified AnnexQueue
-import Utility
+import Utility.SafeCommand
+import Utility.Path
 import LocationLog
 import Types
 import Content
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 8b8d7e364d..fadae0e5a9 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -12,13 +12,11 @@ import System.Directory
 import System.Exit
 
 import Command
-import Messages
-import Types
-import Utility
+import Utility.SafeCommand
 import qualified Git
 import qualified Annex
 import qualified Command.Unannex
-import qualified Command.Init
+import Init
 import qualified Branch
 import Content
 import Locations
@@ -47,7 +45,7 @@ perform = next cleanup
 cleanup :: CommandCleanup
 cleanup = do
 	g <- Annex.gitRepo
-	gitPreCommitHookUnWrite g
+	uninitialize
 	mapM_ removeAnnex =<< getKeysPresent
 	liftIO $ removeDirectoryRecursive (gitAnnexDir g)
 	-- avoid normal shutdown
@@ -55,14 +53,3 @@ cleanup = do
 	liftIO $ do
 		Git.run g "branch" [Param "-D", Param Branch.name]
 		exitSuccess
-
-gitPreCommitHookUnWrite :: Git.Repo -> Annex ()
-gitPreCommitHookUnWrite repo = do
-	let hook = Command.Init.preCommitHook repo
-	whenM (liftIO $ doesFileExist hook) $ do
-		c <- liftIO $ readFile hook
-		if c == Command.Init.preCommitScript
-			then liftIO $ removeFile hook
-			else warning $ "pre-commit hook (" ++ hook ++ 
-				") contents modified; not deleting." ++
-				" Edit it to remove call to git annex."
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index 280eff9de7..0daf1b3218 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -16,8 +16,9 @@ import Types
 import Messages
 import Locations
 import Content
+import Utility.Conditional
 import Utility.CopyFile
-import Utility
+import Utility.Path
 
 command :: [Command]
 command =
diff --git a/Command/Unused.hs b/Command/Unused.hs
index e7065b3c36..6a62cde5f8 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -5,6 +5,8 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
+{-# LANGUAGE BangPatterns #-}
+
 module Command.Unused where
 
 import Control.Monad (filterM, unless, forM_)
@@ -78,9 +80,12 @@ checkRemoteUnused' r = do
 		showLongNote $ remoteUnusedMsg r list
 		showLongNote "\n"
 	where
+		{- This should run strictly to avoid the filterM
+		 - building many thunks containing keyLocations data. -}
 		isthere k = do
 			us <- keyLocations k
-			return $ uuid `elem` us
+			let !there = uuid `elem` us
+			return there
 		uuid = Remote.uuid r
 
 writeUnusedFile :: FilePath -> [(Int, Key)] -> Annex ()
diff --git a/Command/Version.hs b/Command/Version.hs
index 2392c9bf6b..1ff829a22a 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -12,7 +12,7 @@ import Data.String.Utils
 import Data.Maybe
 
 import Command
-import qualified SysConfig
+import qualified Build.SysConfig as SysConfig
 import Version
 
 command :: [Command]
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 314fef7826..f80c823b70 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -33,7 +33,7 @@ perform key = do
 	if null uuids
 		then stop
 		else do
-			pp <- prettyPrintUUIDs uuids
+			pp <- prettyPrintUUIDs "whereis" uuids
 			showLongNote pp
 			showOutput
 			next $ return True
diff --git a/Config.hs b/Config.hs
index 9cbf2d52fd..b4f4c0b922 100644
--- a/Config.hs
+++ b/Config.hs
@@ -9,11 +9,14 @@ module Config where
 
 import Data.Maybe
 import Control.Monad.State (liftIO)
+import Control.Applicative
+import System.Cmd.Utils
 
 import qualified Git
 import qualified Annex
 import Types
 import Utility
+import Utility.SafeCommand
 
 type ConfigKey = String
 
@@ -37,17 +40,22 @@ getConfig r key def = do
 remoteConfig :: Git.Repo -> ConfigKey -> String
 remoteConfig r key = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-" ++ key
 
-{- Calculates cost for a remote.
- -
- - The default cost is 100 for local repositories, and 200 for remote
- - repositories; it can also be configured by remote.<name>.annex-cost
+{- Calculates cost for a remote. Either the default, or as configured 
+ - by remote.<name>.annex-cost, or if remote.<name>.annex-cost-command
+ - is set and prints a number, that is used.
  -}
 remoteCost :: Git.Repo -> Int -> Annex Int
 remoteCost r def = do
-	c <- getConfig r "cost" ""
-	if not $ null c
-		then return $ read c
-		else return def
+	cmd <- getConfig r "cost-command" ""
+	safeparse <$> if not $ null cmd
+			then liftIO $ snd <$> pipeFrom "sh" ["-c", cmd]
+			else getConfig r "cost" ""
+	where
+		safeparse v
+			| null ws = def
+			| otherwise = fromMaybe def $ readMaybe $ head ws
+			where
+				ws = words v
 
 cheapRemoteCost :: Int
 cheapRemoteCost = 100
diff --git a/Content.hs b/Content.hs
index c63042dfb5..e4bbee5282 100644
--- a/Content.hs
+++ b/Content.hs
@@ -23,11 +23,10 @@ module Content (
 	saveState
 ) where
 
-import System.IO.Error (try)
 import System.Directory
 import Control.Monad.State (liftIO)
 import System.Path
-import Control.Monad (when, filterM)
+import Control.Monad
 import System.Posix.Files
 import System.FilePath
 import Data.Maybe
@@ -41,7 +40,9 @@ import qualified Annex
 import qualified AnnexQueue
 import qualified Branch
 import Utility
-import StatFS
+import Utility.Conditional
+import Utility.StatFS
+import Utility.Path
 import Types.Key
 import Utility.DataUnits
 import Config
@@ -252,15 +253,8 @@ getKeysPresent' dir = do
 			levela <- dirContents dir
 			levelb <- mapM dirContents levela
 			contents <- mapM dirContents (concat levelb)
-			files <- filterM present (concat contents)
+			let files = concat contents
 			return $ mapMaybe (fileKey . takeFileName) files
-	where
-		present d = do
-			result <- try $
-				getFileStatus $ d </> takeFileName d
-			case result of
-				Right s -> return $ isRegularFile s
-				Left _ -> return False
 
 {- Things to do to record changes to content. -}
 saveState :: Annex ()
diff --git a/Crypto.hs b/Crypto.hs
index 4fc41ede04..d789b44556 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -38,6 +38,7 @@ import System.IO
 import System.Posix.IO
 import System.Posix.Types
 import System.Posix.Process
+import Control.Applicative
 import Control.Concurrent
 import Control.Exception (finally)
 import System.Exit
@@ -48,6 +49,7 @@ import Types.Key
 import Types.Remote
 import Utility
 import Utility.Base64
+import Utility.SafeCommand
 import Types.Crypto
 
 {- The first half of a Cipher is used for HMAC; the remainder
@@ -135,7 +137,7 @@ encryptCipher (Cipher c) (KeyIds ks) = do
 {- Decrypting an EncryptedCipher is expensive; the Cipher should be cached. -}
 decryptCipher :: RemoteConfig -> EncryptedCipher -> IO Cipher
 decryptCipher _ (EncryptedCipher encipher _) = 
-	return . Cipher =<< gpgPipeStrict decrypt encipher
+	Cipher <$> gpgPipeStrict decrypt encipher
 	where
 		decrypt = [ Param "--decrypt" ]
 
diff --git a/Git.hs b/Git.hs
index 9b7ac7ea91..ab43504e1a 100644
--- a/Git.hs
+++ b/Git.hs
@@ -17,6 +17,7 @@ module Git (
 	localToUrl,
 	repoIsUrl,
 	repoIsSsh,
+	repoIsHttp,
 	repoIsLocalBare,
 	repoDescribe,
 	repoLocation,
@@ -62,6 +63,7 @@ module Git (
 ) where
 
 import Control.Monad (unless, when)
+import Control.Applicative
 import System.Directory
 import System.FilePath
 import System.Posix.Directory
@@ -84,6 +86,9 @@ import System.Exit
 import System.Posix.Env (setEnv, unsetEnv, getEnv)
 
 import Utility
+import Utility.Path
+import Utility.Conditional
+import Utility.SafeCommand
 
 {- There are two types of repositories; those on local disk and those
  - accessed via an URL. -}
@@ -206,6 +211,13 @@ repoIsSsh Repo { location = Url url }
 	| otherwise = False
 repoIsSsh _ = False
 
+repoIsHttp :: Repo -> Bool
+repoIsHttp Repo { location = Url url } 
+	| uriScheme url == "http:" = True
+	| uriScheme url == "https:" = True
+	| otherwise = False
+repoIsHttp _ = False
+
 configAvail ::Repo -> Bool
 configAvail Repo { config = c } = c /= M.empty
 
@@ -239,11 +251,11 @@ attributes repo
 	| configBare repo = workTree repo ++ "/info/.gitattributes"
 	| otherwise = workTree repo ++ "/.gitattributes"
 
-{- Path to a repository's .git directory, relative to its workTree. -}
+{- Path to a repository's .git directory. -}
 gitDir :: Repo -> String
 gitDir repo
-	| configBare repo = ""
-	| otherwise = ".git"
+	| configBare repo = workTree repo
+	| otherwise = workTree repo </> ".git"
 
 {- Path to a repository's --work-tree, that is, its top.
  -
@@ -337,10 +349,10 @@ urlAuthPart _ repo = assertUrl repo $ error "internal"
 
 {- Constructs a git command line operating on the specified repo. -}
 gitCommandLine :: Repo -> [CommandParam] -> [CommandParam]
-gitCommandLine repo@(Repo { location = Dir d} ) params =
+gitCommandLine repo@(Repo { location = Dir _ } ) params =
 	-- force use of specified repo via --git-dir and --work-tree
-	[ Param ("--git-dir=" ++ d ++ "/" ++ gitDir repo)
-	, Param ("--work-tree=" ++ d)
+	[ Param ("--git-dir=" ++ gitDir repo)
+	, Param ("--work-tree=" ++ workTree repo)
 	] ++ params
 gitCommandLine repo _ = assertLocal repo $ error "internal"
 
@@ -435,7 +447,7 @@ commit g message newref parentrefs = do
 		pipeWriteRead g (map Param $ ["commit-tree", tree] ++ ps) message
 	run g "update-ref" [Param newref, Param sha]
 	where
-		ignorehandle a = return . snd =<< a
+		ignorehandle a = snd <$> a
 		ps = concatMap (\r -> ["-p", r]) parentrefs
 
 {- Reads null terminated output of a git command (as enabled by the -z 
diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs
index 23b383a09d..1ecbb029b5 100644
--- a/Git/LsFiles.hs
+++ b/Git/LsFiles.hs
@@ -16,7 +16,7 @@ module Git.LsFiles (
 ) where
 
 import Git
-import Utility
+import Utility.SafeCommand
 
 {- Scans for files that are checked into git at the specified locations. -}
 inRepo :: Repo -> [FilePath] -> IO [FilePath]
diff --git a/Git/Queue.hs b/Git/Queue.hs
index 0016be4727..25b9ffad08 100644
--- a/Git/Queue.hs
+++ b/Git/Queue.hs
@@ -19,15 +19,15 @@ import System.IO
 import System.Cmd.Utils
 import Data.String.Utils
 import Control.Monad (forM_)
-import Utility
+import Utility.SafeCommand
 
 import Git
 
 {- An action to perform in a git repository. The file to act on
  - is not included, and must be able to be appended after the params. -}
-data Action = Action {
-		getSubcommand :: String,
-		getParams :: [CommandParam]
+data Action = Action
+	{ getSubcommand :: String
+	, getParams :: [CommandParam]
 	} deriving (Show, Eq, Ord)
 
 {- A queue of actions to perform (in any order) on a git repository,
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index b0da071703..a5bcbeac4a 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -18,7 +18,7 @@ import Data.Maybe
 import Data.String.Utils
 
 import Git
-import Utility
+import Utility.SafeCommand
 
 {- Performs a union merge between two branches, staging it in the index.
  - Any previously staged changes in the index will be lost.
diff --git a/Init.hs b/Init.hs
new file mode 100644
index 0000000000..2067c524cf
--- /dev/null
+++ b/Init.hs
@@ -0,0 +1,89 @@
+{- git-annex repository initialization
+ -
+ - Copyright 2011 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Init (
+	ensureInitialized,
+	initialize,
+	uninitialize
+) where
+
+import Control.Monad.State (liftIO)
+import Control.Monad (unless)
+import System.Directory
+
+import qualified Annex
+import qualified Git
+import qualified Branch
+import Version
+import Messages
+import Types
+import Utility
+import Utility.Conditional
+import UUID
+
+initialize :: Annex ()
+initialize = do
+	prepUUID
+	Branch.create
+	setVersion
+	gitPreCommitHookWrite
+
+uninitialize :: Annex ()
+uninitialize = do
+	gitPreCommitHookUnWrite
+
+{- Will automatically initialize if there is already a git-annex
+   branch from somewhere. Otherwise, require a manual init
+   to avoid git-annex accidentially being run in git
+   repos that did not intend to use it. -}
+ensureInitialized :: Annex ()
+ensureInitialized = getVersion >>= maybe needsinit checkVersion
+	where
+		needsinit = do
+			annexed <- Branch.hasSomeBranch
+			if annexed
+				then initialize
+				else error "First run: git-annex init"
+
+{- set up a git pre-commit hook, if one is not already present -}
+gitPreCommitHookWrite :: Annex ()
+gitPreCommitHookWrite = unlessBare $ do
+	hook <- preCommitHook
+	exists <- liftIO $ doesFileExist hook
+	if exists
+		then warning $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring"
+		else liftIO $ do
+			viaTmp writeFile hook preCommitScript
+			p <- getPermissions hook
+			setPermissions hook $ p {executable = True}
+
+gitPreCommitHookUnWrite :: Annex ()
+gitPreCommitHookUnWrite = unlessBare $ do
+	hook <- preCommitHook
+	whenM (liftIO $ doesFileExist hook) $ do
+		c <- liftIO $ readFile hook
+		if c == preCommitScript
+			then liftIO $ removeFile hook
+			else warning $ "pre-commit hook (" ++ hook ++ 
+				") contents modified; not deleting." ++
+				" Edit it to remove call to git annex."
+
+unlessBare :: Annex () -> Annex ()
+unlessBare a = do
+	g <- Annex.gitRepo
+	unless (Git.repoIsLocalBare g) a
+
+preCommitHook :: Annex FilePath
+preCommitHook = do
+	g <- Annex.gitRepo
+	return $ Git.gitDir g ++ "/hooks/pre-commit"
+
+preCommitScript :: String
+preCommitScript = 
+		"#!/bin/sh\n" ++
+		"# automatically configured by git-annex\n" ++ 
+		"git annex pre-commit .\n"
diff --git a/LocationLog.hs b/LocationLog.hs
index 768483fa1b..fa660c8b67 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -24,6 +24,7 @@ module LocationLog (
 
 import System.FilePath
 import Control.Monad (when)
+import Control.Applicative
 import Data.Maybe
 
 import qualified Git
@@ -49,7 +50,7 @@ keyLocations key = currentLog $ logFile key
 {- Finds all keys that have location log information.
  - (There may be duplicate keys in the list.) -}
 loggedKeys :: Annex [Key]
-loggedKeys = return . mapMaybe (logFileKey . takeFileName) =<< Branch.files
+loggedKeys = mapMaybe (logFileKey . takeFileName) <$> Branch.files
 
 {- The filename of the log file for a given key. -}
 logFile :: Key -> String
diff --git a/Makefile b/Makefile
index ccf98f6252..3f36928afb 100644
--- a/Makefile
+++ b/Makefile
@@ -8,12 +8,11 @@ GHCMAKE=ghc $(GHCFLAGS) --make
 
 bins=git-annex git-annex-shell git-union-merge
 mans=git-annex.1 git-annex-shell.1 git-union-merge.1
+sources=Build/SysConfig.hs Utility/StatFS.hs Utility/Touch.hs Remote/S3.hs
 
 all: $(bins) $(mans) docs
 
-sources: SysConfig.hs StatFS.hs Touch.hs Remote/S3.hs
-
-SysConfig.hs: configure.hs TestConfig.hs
+Build/SysConfig.hs: configure.hs Build/TestConfig.hs
 	$(GHCMAKE) configure
 	./configure
 
@@ -30,7 +29,9 @@ Remote/S3.o: Remote/S3.hs
 		echo "** building without S3 support"; \
 	fi
 
-$(bins): SysConfig.hs Touch.hs StatFS.hs Remote/S3.o
+sources: $(sources)
+
+$(bins): sources
 	$(GHCMAKE) $@
 
 git-annex.1: doc/git-annex.mdwn
@@ -54,7 +55,9 @@ test: $(bins)
 	@if ! $(GHCMAKE) -O0 test; then \
 		echo "** not running test suite" >&2; \
 	else \
-		./test; \
+		if ! ./test; then \
+			echo "** test suite failed!" >&2; \
+		fi; \
 	fi
 
 testcoverage: $(bins)
@@ -82,8 +85,7 @@ docs: $(mans)
 		--exclude='news/.*'
 
 clean:
-	rm -rf build $(bins) $(mans) test configure  *.tix .hpc \
-		StatFS.hs Touch.hs SysConfig.hs Remote/S3.hs
+	rm -rf build $(bins) $(mans) test configure  *.tix .hpc $(sources)
 	rm -rf doc/.ikiwiki html dist
 	find . \( -name \*.o -or -name \*.hi \) -exec rm {} \;
 
diff --git a/Messages.hs b/Messages.hs
index 36f0b89c5c..4922519819 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -5,28 +5,40 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Messages where
+module Messages (
+	showStart,
+	showNote,
+	showAction,
+	showProgress,
+	showSideAction,
+	showOutput,
+	showLongNote,
+	showEndOk,
+	showEndFail,
+	showEndResult,
+	showErr,
+	warning,
+	indent,
+	maybeShowJSON,
+	setupConsole
+) where
 
 import Control.Monad.State (liftIO)
 import System.IO
-import Control.Monad (unless)
 import Data.String.Utils
+import Text.JSON
 
 import Types
 import qualified Annex
-
-verbose :: Annex () -> Annex ()
-verbose a = do
-	q <- Annex.getState Annex.quiet
-	unless q a
+import qualified Messages.JSON as JSON
 
 showStart :: String -> String -> Annex ()
-showStart command file = verbose $ liftIO $ do
+showStart command file = handle (JSON.start command file) $ do
 	putStr $ command ++ " " ++ file ++ " "
 	hFlush stdout
 
 showNote :: String -> Annex ()
-showNote s = verbose $ liftIO $ do
+showNote s = handle (JSON.note s) $ do
 	putStr $ "(" ++ s ++ ") "
 	hFlush stdout
 
@@ -34,40 +46,44 @@ showAction :: String -> Annex ()
 showAction s = showNote $ s ++ "..."
 
 showProgress :: Annex ()
-showProgress = verbose $ liftIO $ do
+showProgress = handle q $ do
 	putStr "."
 	hFlush stdout
 
 showSideAction :: String -> Annex ()
-showSideAction s = verbose $ liftIO $ putStrLn $ "(" ++ s ++ "...)"
+showSideAction s = handle q $ putStrLn $ "(" ++ s ++ "...)"
 
 showOutput :: Annex ()
-showOutput = verbose $ liftIO $ putStr "\n"
+showOutput = handle q $ putStr "\n"
 
 showLongNote :: String -> Annex ()
-showLongNote s = verbose $ liftIO $ putStr $ '\n' : indent s
+showLongNote s = handle (JSON.note s) $ putStr $ '\n' : indent s
 
 showEndOk :: Annex ()
-showEndOk = verbose $ liftIO $ putStrLn "ok"
+showEndOk = showEndResult True
 
 showEndFail :: Annex ()
-showEndFail = verbose $ liftIO $ putStrLn "failed"
+showEndFail = showEndResult False
 
 showEndResult :: Bool -> Annex ()
-showEndResult True = showEndOk
-showEndResult False = showEndFail
+showEndResult b = handle (JSON.end b) $ putStrLn msg
+	where
+		msg
+			| b = "ok"
+			| otherwise = "failed"
 
 showErr :: (Show a) => a -> Annex ()
-showErr e = liftIO $ do
-	hFlush stdout
-	hPutStrLn stderr $ "git-annex: " ++ show e
+showErr e = warning' $ "git-annex: " ++ show e
 
 warning :: String -> Annex ()
-warning w = do
-	verbose $ liftIO $ putStr "\n"
+warning w = warning' (indent w)
+
+warning' :: String -> Annex ()
+warning' w = do
+	handle q $ putStr "\n"
 	liftIO $ do
 		hFlush stdout
-		hPutStrLn stderr $ indent w
+		hPutStrLn stderr w
 
 indent :: String -> String
 indent s = join "\n" $ map (\l -> "  " ++ l) $ lines s
@@ -84,3 +100,18 @@ setupConsole :: IO ()
 setupConsole = do
 	hSetBinaryMode stdout True
 	hSetBinaryMode stderr True
+
+handle :: IO () -> IO () -> Annex ()
+handle json normal = do
+	output <- Annex.getState Annex.output
+	case output of
+		Annex.NormalOutput -> liftIO normal
+		Annex.QuietOutput -> q
+		Annex.JSONOutput -> liftIO json
+
+{- Shows a JSON value only when in json mode. -}
+maybeShowJSON :: JSON a => [(String, a)] -> Annex ()
+maybeShowJSON v = handle (JSON.add v) q
+
+q :: Monad m => m ()
+q = return ()
diff --git a/Messages/JSON.hs b/Messages/JSON.hs
new file mode 100644
index 0000000000..fb95f550e8
--- /dev/null
+++ b/Messages/JSON.hs
@@ -0,0 +1,29 @@
+{- git-annex JSON output
+ -
+ - Copyright 2011 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Messages.JSON (
+	start,
+	end,
+	note,
+	add
+) where
+
+import Text.JSON
+
+import qualified Utility.JSONStream as Stream
+
+start :: String -> String -> IO ()
+start command file = putStr $ Stream.start [("command", command), ("file", file)]
+
+end :: Bool -> IO ()
+end b = putStr $ Stream.add [("success", b)] ++ Stream.end
+
+note :: String -> IO ()
+note s = add [("note", s)]
+
+add :: JSON a => [(String, a)] -> IO ()
+add v = putStr $ Stream.add v
diff --git a/Options.hs b/Options.hs
index 7f78f44f62..e0ca48c01b 100644
--- a/Options.hs
+++ b/Options.hs
@@ -26,10 +26,12 @@ commonOptions =
 		"allow actions that may lose annexed data"
 	, Option ['F'] ["fast"] (NoArg (setfast True))
 		"avoid slow operations"
-	, Option ['q'] ["quiet"] (NoArg (setquiet True))
+	, Option ['q'] ["quiet"] (NoArg (setoutput Annex.QuietOutput))
 		"avoid verbose output"
-	, Option ['v'] ["verbose"] (NoArg (setquiet False))
+	, Option ['v'] ["verbose"] (NoArg (setoutput Annex.NormalOutput))
 		"allow verbose output (default)"
+	, Option ['j'] ["json"] (NoArg (setoutput Annex.JSONOutput))
+		"enable JSON output"
 	, Option ['d'] ["debug"] (NoArg (setdebug))
 		"show debug messages"
 	, Option ['b'] ["backend"] (ReqArg setforcebackend paramName)
@@ -38,7 +40,7 @@ commonOptions =
 	where
 		setforce v = Annex.changeState $ \s -> s { Annex.force = v }
 		setfast v = Annex.changeState $ \s -> s { Annex.fast = v }
-		setquiet v = Annex.changeState $ \s -> s { Annex.quiet = v }
+		setoutput v = Annex.changeState $ \s -> s { Annex.output = v }
 		setforcebackend v = Annex.changeState $ \s -> s { Annex.forcebackend = Just v }
 		setdebug = liftIO $ updateGlobalLogger rootLoggerName $
 			setLevel DEBUG
diff --git a/PresenceLog.hs b/PresenceLog.hs
index 9c516a8db1..e0c8729979 100644
--- a/PresenceLog.hs
+++ b/PresenceLog.hs
@@ -15,10 +15,12 @@ module PresenceLog (
 	LogStatus(..),
 	addLog,
 	readLog,
+	parseLog,
 	writeLog,
 	logNow,
 	compactLog,
-	currentLog
+	currentLog,
+	LogLine
 ) where
 
 import Data.Time.Clock.POSIX
@@ -26,6 +28,7 @@ import Data.Time
 import System.Locale
 import qualified Data.Map as Map
 import Control.Monad.State (liftIO)
+import Control.Applicative
 
 import qualified Branch
 import Types
@@ -79,7 +82,7 @@ addLog file line = do
 {- Reads a log file.
  - Note that the LogLines returned may be in any order. -}
 readLog :: FilePath -> Annex [LogLine]
-readLog file = return . parseLog =<< Branch.get file
+readLog file = parseLog <$> Branch.get file
 
 parseLog :: String -> [LogLine]
 parseLog s = filter parsable $ map read $ lines s
diff --git a/Remote.hs b/Remote.hs
index 1a5006f6fb..e54d2e2334 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -29,11 +29,14 @@ module Remote (
 	forceTrust
 ) where
 
-import Control.Monad (filterM, liftM2)
+import Control.Monad (filterM)
 import Data.List
 import qualified Data.Map as M
 import Data.String.Utils
 import Data.Maybe
+import Control.Applicative
+import Text.JSON
+import Text.JSON.Generic
 
 import Types
 import Types.Remote
@@ -111,30 +114,40 @@ nameToUUID "." = getUUID =<< Annex.gitRepo -- special case for current repo
 nameToUUID n = do
 	res <- byName' n
 	case res of
-		Left e -> return . fromMaybe (error e) =<< byDescription
+		Left e -> fromMaybe (error e) <$> byDescription
 		Right r -> return $ uuid r
 	where
-		byDescription = return . M.lookup n . invertMap =<< uuidMap
+		byDescription = M.lookup n . invertMap <$> uuidMap
 		invertMap = M.fromList . map swap . M.toList
 		swap (a, b) = (b, a)
 
-{- Pretty-prints a list of UUIDs of remotes. -}
-prettyPrintUUIDs :: [UUID] -> Annex String
-prettyPrintUUIDs uuids = do
+{- Pretty-prints a list of UUIDs of remotes, for human display.
+ -
+ - Shows descriptions from the uuid log, falling back to remote names,
+ - as some remotes may not be in the uuid log.
+ -
+ - When JSON is enabled, also generates a machine-readable description
+ - of the UUIDs. -}
+prettyPrintUUIDs :: String -> [UUID] -> Annex String
+prettyPrintUUIDs desc uuids = do
 	here <- getUUID =<< Annex.gitRepo
-	-- Show descriptions from the uuid log, falling back to remote names,
-	-- as some remotes may not be in the uuid log
-	m <- liftM2 M.union uuidMap $
-		return . M.fromList . map (\r -> (uuid r, name r)) =<< genList
-	return $ unwords $ map (\u -> "\t" ++ prettify m u here ++ "\n") uuids
+	m <- M.union <$> uuidMap <*> availMap
+	maybeShowJSON [(desc, map (jsonify m here) uuids)]
+	return $ unwords $ map (\u -> "\t" ++ prettify m here u ++ "\n") uuids
 	where
-		prettify m u here = base ++ ishere
+		availMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList
+		findlog m u = M.findWithDefault "" u m
+		prettify m here u = base ++ ishere
 			where
 				base = if not $ null $ findlog m u
 					then u ++ "  -- " ++ findlog m u
 					else u
 				ishere = if here == u then " <-- here" else ""
-		findlog m u = M.findWithDefault "" u m
+		jsonify m here u = toJSObject
+			[ ("uuid", toJSON u)
+			, ("description", toJSON $ findlog m u)
+			, ("here", toJSON $ here == u)
+			]
 
 {- Filters a list of remotes to ones that have the listed uuids. -}
 remotesWithUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
@@ -147,7 +160,7 @@ remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs
 {- Cost ordered lists of remotes that the LocationLog indicate may have a key.
  -}
 keyPossibilities :: Key -> Annex [Remote Annex]
-keyPossibilities key = return . fst =<< keyPossibilities' False key
+keyPossibilities key = fst <$> keyPossibilities' False key
 
 {- Cost ordered lists of remotes that the LocationLog indicate may have a key.
  -
@@ -185,8 +198,8 @@ showLocations key exclude = do
 	untrusteduuids <- trustGet UnTrusted
 	let uuidswanted = filteruuids uuids (u:exclude++untrusteduuids) 
 	let uuidsskipped = filteruuids uuids (u:exclude++uuidswanted)
-	ppuuidswanted <- Remote.prettyPrintUUIDs uuidswanted
-	ppuuidsskipped <- Remote.prettyPrintUUIDs uuidsskipped
+	ppuuidswanted <- Remote.prettyPrintUUIDs "wanted" uuidswanted
+	ppuuidsskipped <- Remote.prettyPrintUUIDs "skipped" uuidsskipped
 	showLongNote $ message ppuuidswanted ppuuidsskipped
 	where
 		filteruuids l x = filter (`notElem` x) l
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 1023cda186..ebb4b10a5b 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -29,10 +29,12 @@ import UUID
 import Locations
 import Config
 import Utility
+import Utility.Conditional
+import Utility.SafeCommand
 import Messages
-import Remote.Ssh
-import Remote.Special
-import Remote.Encryptable
+import Utility.Ssh
+import Remote.Helper.Special
+import Remote.Helper.Encryptable
 import Crypto
 
 type BupRepo = String
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 235f613000..7ddb60462f 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -27,8 +27,10 @@ import Utility.CopyFile
 import Config
 import Content
 import Utility
-import Remote.Special
-import Remote.Encryptable
+import Utility.Conditional
+import Utility.Path
+import Remote.Helper.Special
+import Remote.Helper.Encryptable
 import Crypto
 
 remote :: RemoteType Annex
diff --git a/Remote/Git.hs b/Remote/Git.hs
index de51c891e2..fadd48a036 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -12,6 +12,7 @@ import Control.Monad.State (liftIO)
 import qualified Data.Map as M
 import System.Cmd.Utils
 import System.Posix.Files
+import System.IO
 
 import Types
 import Types.Remote
@@ -24,8 +25,12 @@ import qualified Content
 import Messages
 import Utility.CopyFile
 import Utility.RsyncFile
-import Remote.Ssh
+import Utility.Ssh
+import Utility.SafeCommand
+import Utility.Path
+import qualified Utility.Url as Url
 import Config
+import Init
 
 remote :: RemoteType Annex
 remote = RemoteType {
@@ -75,8 +80,11 @@ tryGitConfigRead :: Git.Repo -> Annex Git.Repo
 tryGitConfigRead r 
 	| not $ M.null $ Git.configMap r = return r -- already read
 	| Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" []
+	| Git.repoIsHttp r = store $ safely $ geturlconfig
 	| Git.repoIsUrl r = return r
-	| otherwise = store $ safely $ Git.configRead r
+	| otherwise = store $ safely $ do
+		onLocal r ensureInitialized
+		Git.configRead r
 	where
 		-- Reading config can fail due to IO error or
 		-- for other reasons; catch all possible exceptions.
@@ -85,9 +93,19 @@ tryGitConfigRead r
 			case result of
 				Left _ -> return r
 				Right r' -> return r'
+
 		pipedconfig cmd params = safely $
 			pOpen ReadFromPipe cmd (toCommand params) $
 				Git.hConfigRead r
+
+		geturlconfig = do
+			s <- Url.get (Git.repoLocation r ++ "/config")
+			withTempFile "git-annex.tmp" $ \tmpfile -> \h -> do
+				hPutStr h s
+				hClose h
+				pOpen ReadFromPipe "git" ["config", "--list", "--file", tmpfile] $
+					Git.hConfigRead r
+
 		store a = do
 			r' <- a
 			g <- Annex.gitRepo
@@ -95,6 +113,7 @@ tryGitConfigRead r
 			let g' = Git.remotesAdd g $ exchange l r'
 			Annex.changeState $ \s -> s { Annex.repo = g' }
 			return r'
+
 		exchange [] _ = []
 		exchange (old:ls) new =
 			if Git.repoRemoteName old == Git.repoRemoteName new
@@ -105,24 +124,34 @@ tryGitConfigRead r
  - If the remote cannot be accessed, returns a Left error.
  -}
 inAnnex :: Git.Repo -> Key -> Annex (Either IOException Bool)
-inAnnex r key = if Git.repoIsUrl r
-		then checkremote
-		else liftIO (try checklocal ::IO (Either IOException Bool))
+inAnnex r key
+	| Git.repoIsHttp r = safely checkhttp
+	| Git.repoIsUrl r = checkremote
+	| otherwise = safely checklocal
 	where
-		checklocal = do
-			-- run a local check inexpensively,
-			-- by making an Annex monad using the remote
-			a <- Annex.new r
-			Annex.eval a (Content.inAnnex key)
+		checklocal = onLocal r (Content.inAnnex key)
 		checkremote = do
 			showAction $ "checking " ++ Git.repoDescribe r
 			inannex <- onRemote r (boolSystem, False) "inannex" 
 				[Param (show key)]
 			return $ Right inannex
-	
+		checkhttp = Url.exists $ keyUrl r key
+		safely a = liftIO (try a ::IO (Either IOException Bool))
+
+{- Runs an action on a local repository inexpensively, by making an annex
+ - monad using that repository. -}
+onLocal :: Git.Repo -> Annex a -> IO a
+onLocal r a = do
+	annex <- Annex.new r
+	Annex.eval annex a
+
+keyUrl :: Git.Repo -> Key -> String
+keyUrl r key = Git.repoLocation r ++ "/" ++ annexLocation key
+
 dropKey :: Git.Repo -> Key -> Annex Bool
-dropKey r key = 
-	onRemote r (boolSystem, False) "dropkey"
+dropKey r key
+	| Git.repoIsHttp r = error "dropping from http repo not supported"
+	| otherwise = onRemote r (boolSystem, False) "dropkey"
 		[ Params "--quiet --force"
 		, Param $ show key
 		]
@@ -132,8 +161,9 @@ copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool
 copyFromRemote r key file
 	| not $ Git.repoIsUrl r = rsyncOrCopyFile r (gitAnnexLocation r key) file
 	| Git.repoIsSsh r = rsyncHelper =<< rsyncParamsRemote r True key file
-	| otherwise = error "copying from non-ssh repo not supported"
-		
+	| Git.repoIsHttp r = Url.download (keyUrl r key) file
+	| otherwise = error "copying from non-ssh, non-http repo not supported"
+
 {- Tries to copy a key's content to a remote's annex. -}
 copyToRemote :: Git.Repo -> Key -> Annex Bool
 copyToRemote r key
@@ -141,13 +171,11 @@ copyToRemote r key
 		g <- Annex.gitRepo
 		let keysrc = gitAnnexLocation g key
 		-- run copy from perspective of remote
-		liftIO $ do
-			a <- Annex.new r
-			Annex.eval a $ do
-				ok <- Content.getViaTmp key $
-					rsyncOrCopyFile r keysrc
-				Content.saveState
-				return ok
+		liftIO $ onLocal r $ do
+			ok <- Content.getViaTmp key $
+				rsyncOrCopyFile r keysrc
+			Content.saveState
+			return ok
 	| Git.repoIsSsh r = do
 		g <- Annex.gitRepo
 		let keysrc = gitAnnexLocation g key
diff --git a/Remote/Encryptable.hs b/Remote/Helper/Encryptable.hs
similarity index 98%
rename from Remote/Encryptable.hs
rename to Remote/Helper/Encryptable.hs
index 66e1738ac2..04041c6553 100644
--- a/Remote/Encryptable.hs
+++ b/Remote/Helper/Encryptable.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Remote.Encryptable where
+module Remote.Helper.Encryptable where
 
 import qualified Data.Map as M
 import Control.Monad.State (liftIO)
diff --git a/Remote/Special.hs b/Remote/Helper/Special.hs
similarity index 95%
rename from Remote/Special.hs
rename to Remote/Helper/Special.hs
index d6f362ce30..b842588c0c 100644
--- a/Remote/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Remote.Special where
+module Remote.Helper.Special where
 
 import qualified Data.Map as M
 import Data.Maybe
@@ -17,7 +17,7 @@ import Types.Remote
 import qualified Git
 import qualified Annex
 import UUID
-import Utility
+import Utility.SafeCommand
 
 {- Special remotes don't have a configured url, so Git.Repo does not
  - automatically generate remotes for them. This looks for a different
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 87f86ffe4f..54b9806ffc 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -28,8 +28,9 @@ import Locations
 import Config
 import Content
 import Utility
-import Remote.Special
-import Remote.Encryptable
+import Utility.SafeCommand
+import Remote.Helper.Special
+import Remote.Helper.Encryptable
 import Crypto
 import Messages
 
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index f073e7bd79..3707966ad6 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -26,11 +26,14 @@ import Locations
 import Config
 import Content
 import Utility
-import Remote.Special
-import Remote.Encryptable
+import Utility.Conditional
+import Remote.Helper.Special
+import Remote.Helper.Encryptable
 import Crypto
 import Messages
 import Utility.RsyncFile
+import Utility.SafeCommand
+import Utility.Path
 
 type RsyncUrl = String
 
@@ -93,10 +96,13 @@ rsyncSetup u c = do
 	return c'
 
 rsyncKey :: RsyncOpts -> Key -> String
-rsyncKey o k = rsyncUrl o </> hashDirMixed k </> f </> f
+rsyncKey o k = rsyncUrl o </> hashDirMixed k </> shellEscape (f </> f)
         where
                 f = keyFile k
 
+rsyncKeyDir :: RsyncOpts -> Key -> String
+rsyncKeyDir o k = rsyncUrl o </> hashDirMixed k </> shellEscape (keyFile k)
+
 store :: RsyncOpts -> Key -> Annex Bool
 store o k = do
 	g <- Annex.gitRepo
@@ -136,7 +142,7 @@ remove o k = withRsyncScratchDir $ \tmp -> do
 		[ Params "--delete --recursive"
 		, partialParams
 		, Param $ addTrailingPathSeparator dummy
-		, Param $ parentDir $ rsyncKey o k
+		, Param $ rsyncKeyDir o k
 		]
 
 checkPresent :: Git.Repo -> RsyncOpts -> Key -> Annex (Either IOException Bool)
@@ -147,8 +153,7 @@ checkPresent r o k = do
 	res <- liftIO $ boolSystem "sh" [Param "-c", Param cmd]
 	return $ Right res
 	where
-		cmd = "rsync --quiet " ++ testfile ++ " 2>/dev/null"
-		testfile = shellEscape $ rsyncKey o k
+		cmd = "rsync --quiet " ++ shellEscape (rsyncKey o k) ++ " 2>/dev/null"
 
 {- Rsync params to enable resumes of sending files safely,
  - ensure that files are only moved into place once complete
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index e4dcc2a71d..456a77f0e4 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -33,8 +33,8 @@ import UUID
 import Messages
 import Locations
 import Config
-import Remote.Special
-import Remote.Encryptable
+import Remote.Helper.Special
+import Remote.Helper.Encryptable
 import Crypto
 import Content
 import Utility.Base64
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 2f8fac23b5..3695bb1ab3 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -7,28 +7,25 @@
 
 module Remote.Web (
 	remote,
-	setUrl,
-	download
+	setUrl
 ) where
 
 import Control.Monad.State (liftIO)
 import Control.Exception
 import System.FilePath
-import Network.Browser
-import Network.HTTP
-import Network.URI
 
 import Types
 import Types.Remote
 import qualified Git
 import qualified Annex
 import Messages
-import Utility
 import UUID
 import Config
 import PresenceLog
 import LocationLog
 import Locations
+import Utility
+import qualified Utility.Url as Url
 
 type URLString = String
 
@@ -67,10 +64,17 @@ gen r _ _ =
 {- The urls for a key are stored in remote/web/hash/key.log 
  - in the git-annex branch. -}
 urlLog :: Key -> FilePath
-urlLog key = "remote/web" </> hashDirLower key </> show key ++ ".log"
+urlLog key = "remote/web" </> hashDirLower key </> keyFile key ++ ".log"
+oldurlLog :: Key -> FilePath
+{- A bug used to store the urls elsewhere. -}
+oldurlLog key = "remote/web" </> hashDirLower key </> show key ++ ".log"
 
 getUrls :: Key -> Annex [URLString]
-getUrls key = currentLog (urlLog key)
+getUrls key = do
+	us <- currentLog (urlLog key)
+	if null us
+		then currentLog (oldurlLog key)
+		else return us
 
 {- Records a change in an url for a key. -}
 setUrl :: Key -> URLString -> LogStatus -> Annex ()
@@ -83,9 +87,12 @@ setUrl key url status = do
 	logChange g key webUUID (if null us then InfoMissing else InfoPresent)
 
 downloadKey :: Key -> FilePath -> Annex Bool
-downloadKey key file = do
-	us <- getUrls key
-	download us file
+downloadKey key file = get =<< getUrls key
+	where
+		get [] = do
+			warning "no known url"
+			return False
+		get urls = anyM (`Url.download` file) urls
 
 uploadKey :: Key -> Annex Bool
 uploadKey _ = do
@@ -107,28 +114,5 @@ checkKey' :: [URLString] -> Annex Bool
 checkKey' [] = return False
 checkKey' (u:us) = do
 	showAction $ "checking " ++ u
-	e <- liftIO $ urlexists u
+	e <- liftIO $ Url.exists u
 	if e then return e else checkKey' us
-
-urlexists :: URLString -> IO Bool
-urlexists url =
-	case parseURI url of
-		Nothing -> return False
-		Just u -> do
-			(_, r) <- Network.Browser.browse $ do
-				setErrHandler ignore
-				setOutHandler ignore
-				setAllowRedirects True
-				request (mkRequest HEAD u :: Request_String)
-			case rspCode r of
-				(2,_,_) -> return True
-				_ -> return False
-	where
-		ignore = const $ return ()
-
-download :: [URLString] -> FilePath -> Annex Bool
-download [] _ = return False
-download (url:us) file = do
-	showOutput -- make way for curl progress bar
-	ok <- liftIO $ boolSystem "curl" [Params "-L -C - -# -o", File file, File url]
-	if ok then return ok else download us file
diff --git a/RemoteLog.hs b/RemoteLog.hs
index 69a82f4987..620c0d8757 100644
--- a/RemoteLog.hs
+++ b/RemoteLog.hs
@@ -19,6 +19,7 @@ import Data.List
 import qualified Data.Map as M
 import Data.Maybe
 import Data.Char
+import Control.Applicative
 
 import qualified Branch
 import Types
@@ -40,7 +41,7 @@ configSet u c = do
 
 {- Map of remotes by uuid containing key/value config maps. -}
 readRemoteLog :: Annex (M.Map UUID RemoteConfig)
-readRemoteLog = return . remoteLogParse =<< Branch.get remoteLog
+readRemoteLog = remoteLogParse <$> Branch.get remoteLog
 
 remoteLogParse :: String -> M.Map UUID RemoteConfig
 remoteLogParse s =
diff --git a/UUID.hs b/UUID.hs
index cfa519baa3..fa71bed391 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -33,7 +33,7 @@ import qualified Branch
 import Types
 import Types.UUID
 import qualified Annex
-import qualified SysConfig
+import qualified Build.SysConfig as SysConfig
 import Config
 
 configkey :: String
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index c41310880f..9c3fd99595 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -11,6 +11,7 @@ import System.IO.Error (try)
 import System.Directory
 import Control.Monad.State (liftIO)
 import Control.Monad (filterM, forM_, unless)
+import Control.Applicative
 import System.Posix.Files
 import System.FilePath
 import Data.String.Utils
@@ -22,7 +23,7 @@ import Types.Key
 import Content
 import Types
 import Locations
-import LocationLog
+import PresenceLog
 import qualified Annex
 import qualified AnnexQueue
 import qualified Git
@@ -31,6 +32,8 @@ import Backend
 import Messages
 import Version
 import Utility
+import Utility.SafeCommand
+import Utility.Path
 import qualified Upgrade.V2
 
 -- v2 adds hashing of filenames of content and location log files.
@@ -123,7 +126,7 @@ moveLocationLogs = do
 					else return []
 			move (l, k) = do
 				g <- Annex.gitRepo
-				let dest = logFile k
+				let dest = logFile2 g k
 				let dir = Upgrade.V2.gitStateDir g
 				let f = dir </> l
 				liftIO $ createDirectoryIfMissing True (parentDir dest)
@@ -131,9 +134,9 @@ moveLocationLogs = do
 				-- log files that are not checked into git,
 				-- as well as merging with already upgraded
 				-- logs that have been pulled from elsewhere
-				old <- readLog f
-				new <- readLog dest
-				writeLog dest (old++new)
+				old <- liftIO $ readLog1 f
+				new <- liftIO $ readLog1 dest
+				liftIO $ writeLog1 dest (old++new)
 				AnnexQueue.add "add" [Param "--"] [dest]
 				AnnexQueue.add "add" [Param "--"] [f]
 				AnnexQueue.add "rm" [Param "--quiet", Param "-f", Param "--"] [f]
@@ -186,8 +189,11 @@ fileKey1 :: FilePath -> Key
 fileKey1 file = readKey1 $
 	replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file
 
-logFile1 :: Git.Repo -> Key -> String
-logFile1 repo key = Upgrade.V2.gitStateDir repo ++ keyFile1 key ++ ".log"
+writeLog1 :: FilePath -> [LogLine] -> IO ()
+writeLog1 file ls = viaTmp writeFile file (unlines $ map show ls)
+
+readLog1 :: FilePath -> IO [LogLine]
+readLog1 file = catch (parseLog <$> readFileStrict file) (const $ return [])
 
 lookupFile1 :: FilePath -> Annex (Maybe (Key, Backend Annex))
 lookupFile1 file = do
@@ -196,7 +202,7 @@ lookupFile1 file = do
 		Left _ -> return Nothing
 		Right l -> makekey l
 	where
-		getsymlink = return . takeFileName =<< readSymbolicLink file
+		getsymlink = takeFileName <$> readSymbolicLink file
 		makekey l = case maybeLookupBackendName bname of
 			Nothing -> do
 				unless (null kname || null bname ||
@@ -230,3 +236,19 @@ getKeyFilesPresent1' dir = do
 			case result of
 				Right s -> return $ isRegularFile s
 				Left _ -> return False
+
+logFile1 :: Git.Repo -> Key -> String
+logFile1 repo key = Upgrade.V2.gitStateDir repo ++ keyFile1 key ++ ".log"
+
+logFile2 :: Git.Repo -> Key -> String
+logFile2 = logFile' hashDirLower
+
+logFile' :: (Key -> FilePath) -> Git.Repo -> Key -> String
+logFile' hasher repo key =
+	gitStateDir repo ++ hasher key ++ keyFile key ++ ".log"
+
+stateDir :: FilePath
+stateDir = addTrailingPathSeparator $ ".git-annex"
+
+gitStateDir :: Git.Repo -> FilePath
+gitStateDir repo = addTrailingPathSeparator $ Git.workTree repo </> stateDir
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 0b1d69f8e1..ffd0f06535 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -20,6 +20,8 @@ import qualified Git
 import qualified Branch
 import Messages
 import Utility
+import Utility.Conditional
+import Utility.SafeCommand
 import LocationLog
 import Content
 
diff --git a/Utility.hs b/Utility.hs
index dab3b4d0ea..ce17363488 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -6,144 +6,33 @@
  -}
 
 module Utility (
-	CommandParam(..),
-	toCommand,
 	hGetContentsStrict,
 	readFileStrict,
-	parentDir,
-	absPath,
-	absPathFrom,
-	relPathCwdToFile,
-	relPathDirToFile,
-	boolSystem,
-	boolSystemEnv,
-	executeFile,
-	shellEscape,
-	shellUnEscape,
 	unsetFileMode,
 	readMaybe,
 	viaTmp,
+	withTempFile,
 	dirContains,
 	dirContents,
 	myHomeDir,
 	catchBool,
-	whenM,
-	(>>?),
-	unlessM,
-	(>>!),
-	
-	prop_idempotent_shellEscape,
-	prop_idempotent_shellEscape_multiword,
-	prop_parentDir_basics,
-	prop_relPathDirToFile_basics
+	inPath,
+	firstM,
+	anyM
 ) where
 
+import IO (bracket)
 import System.IO
-import System.Exit
-import qualified System.Posix.Process
 import System.Posix.Process hiding (executeFile)
-import System.Posix.Signals
 import System.Posix.Files
 import System.Posix.Types
 import System.Posix.User
-import Data.String.Utils
-import System.Path
 import System.FilePath
 import System.Directory
 import Foreign (complement)
-import Data.List
+import Utility.Path
 import Data.Maybe
-import Control.Monad (liftM2, when, unless)
-import System.Log.Logger
-
-{- A type for parameters passed to a shell command. A command can
- - be passed either some Params (multiple parameters can be included,
- - whitespace-separated, or a single Param (for when parameters contain
- - whitespace), or a File.
- -}
-data CommandParam = Params String | Param String | File FilePath
-	deriving (Eq, Show, Ord)
-
-{- Used to pass a list of CommandParams to a function that runs
- - a command and expects Strings. -}
-toCommand :: [CommandParam] -> [String]
-toCommand = (>>= unwrap)
-	where
-		unwrap (Param s) = [s]
-		unwrap (Params s) = filter (not . null) (split " " s)
-		-- Files that start with a dash are modified to avoid
-		-- the command interpreting them as options.
-		unwrap (File ('-':s)) = ["./-" ++ s]
-		unwrap (File s) = [s]
-
-{- Run a system command, and returns True or False
- - if it succeeded or failed.
- -
- - SIGINT(ctrl-c) is allowed to propigate and will terminate the program.
- -}
-boolSystem :: FilePath -> [CommandParam] -> IO Bool
-boolSystem command params = boolSystemEnv command params Nothing
-
-boolSystemEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO Bool
-boolSystemEnv command params env = do
-	-- Going low-level because all the high-level system functions
-	-- block SIGINT etc. We need to block SIGCHLD, but allow
-	-- SIGINT to do its default program termination.
-	let sigset = addSignal sigCHLD emptySignalSet
-	oldint <- installHandler sigINT Default Nothing
-	oldset <- getSignalMask
-	blockSignals sigset
-	childpid <- forkProcess $ childaction oldint oldset
-	mps <- getProcessStatus True False childpid
-	restoresignals oldint oldset
-	case mps of
-		Just (Exited ExitSuccess) -> return True
-		_ -> return False
-	where
-		restoresignals oldint oldset = do
-			_ <- installHandler sigINT oldint Nothing
-			setSignalMask oldset
-		childaction oldint oldset = do
-			restoresignals oldint oldset
-			executeFile command True (toCommand params) env
-
-{- executeFile with debug logging -}
-executeFile :: FilePath -> Bool -> [String] -> Maybe [(String, String)] -> IO ()
-executeFile c path p e = do
-	debugM "Utility.executeFile" $
-		"Running: " ++ c ++ " " ++ show p ++ " " ++ maybe "" show e
-	System.Posix.Process.executeFile c path p e
-
-{- Escapes a filename or other parameter to be safely able to be exposed to
- - the shell. -}
-shellEscape :: String -> String
-shellEscape f = "'" ++ escaped ++ "'"
-	where
-		-- replace ' with '"'"'
-		escaped = join "'\"'\"'" $ split "'" f
-
-{- Unescapes a set of shellEscaped words or filenames. -}
-shellUnEscape :: String -> [String]
-shellUnEscape [] = []
-shellUnEscape s = word : shellUnEscape rest
-	where
-		(word, rest) = findword "" s
-		findword w [] = (w, "")
-		findword w (c:cs)
-			| c == ' ' = (w, cs)
-			| c == '\'' = inquote c w cs
-			| c == '"' = inquote c w cs
-			| otherwise = findword (w++[c]) cs
-		inquote _ w [] = (w, "")
-		inquote q w (c:cs)
-			| c == q = findword w cs
-			| otherwise = inquote q (w++[c]) cs
-
-{- For quickcheck. -}
-prop_idempotent_shellEscape :: String -> Bool
-prop_idempotent_shellEscape s = [s] == (shellUnEscape . shellEscape) s
-prop_idempotent_shellEscape_multiword :: [String] -> Bool
-prop_idempotent_shellEscape_multiword s = s == (shellUnEscape . unwords . map shellEscape) s
+import Control.Monad (liftM)
 
 {- A version of hgetContents that is not lazy. Ensures file is 
  - all read before it gets closed. -}
@@ -154,82 +43,6 @@ hGetContentsStrict h  = hGetContents h >>= \s -> length s `seq` return s
 readFileStrict :: FilePath -> IO String
 readFileStrict f = readFile f >>= \s -> length s `seq` return s
 
-{- Returns the parent directory of a path. Parent of / is "" -}
-parentDir :: FilePath -> FilePath
-parentDir dir =
-	if not $ null dirs
-	then slash ++ join s (take (length dirs - 1) dirs)
-	else ""
-		where
-			dirs = filter (not . null) $ split s dir
-			slash = if isAbsolute dir then s else ""
-			s = [pathSeparator]
-
-prop_parentDir_basics :: FilePath -> Bool
-prop_parentDir_basics dir
-	| null dir = True
-	| dir == "/" = parentDir dir == ""
-	| otherwise = p /= dir
-	where
-		p = parentDir dir
-
-{- Checks if the first FilePath is, or could be said to contain the second.
- - For example, "foo/" contains "foo/bar". Also, "foo", "./foo", "foo/" etc
- - are all equivilant.
- -}
-dirContains :: FilePath -> FilePath -> Bool
-dirContains a b = a == b || a' == b' || (a'++"/") `isPrefixOf` b'
-	where
-		norm p = fromMaybe "" $ absNormPath p "."
-		a' = norm a
-		b' = norm b
-
-{- Converts a filename into a normalized, absolute path. -}
-absPath :: FilePath -> IO FilePath
-absPath file = do
-	cwd <- getCurrentDirectory
-	return $ absPathFrom cwd file
-
-{- Converts a filename into a normalized, absolute path
- - from the specified cwd. -}
-absPathFrom :: FilePath -> FilePath -> FilePath
-absPathFrom cwd file = fromMaybe bad $ absNormPath cwd file
-	where
-		bad = error $ "unable to normalize " ++ file
-
-{- Constructs a relative path from the CWD to a file.
- -
- - For example, assuming CWD is /tmp/foo/bar:
- -    relPathCwdToFile "/tmp/foo" == ".."
- -    relPathCwdToFile "/tmp/foo/bar" == "" 
- -}
-relPathCwdToFile :: FilePath -> IO FilePath
-relPathCwdToFile f = liftM2 relPathDirToFile getCurrentDirectory (absPath f)
-
-{- Constructs a relative path from a directory to a file.
- -
- - Both must be absolute, and normalized (eg with absNormpath).
- -}
-relPathDirToFile :: FilePath -> FilePath -> FilePath
-relPathDirToFile from to = path
-	where
-		s = [pathSeparator]
-		pfrom = split s from
-		pto = split s to
-		common = map fst $ filter same $ zip pfrom pto
-		same (c,d) = c == d
-		uncommon = drop numcommon pto
-		dotdots = replicate (length pfrom - numcommon) ".."
-		numcommon = length common
-		path = join s $ dotdots ++ uncommon
-
-prop_relPathDirToFile_basics :: FilePath -> FilePath -> Bool
-prop_relPathDirToFile_basics from to
-	| from == to = null r
-	| otherwise = not (null r)
-	where
-		r = relPathDirToFile from to 
-
 {- Removes a FileMode from a file.
  - For example, call with otherWriteMode to chmod o-w -}
 unsetFileMode :: FilePath -> FileMode -> IO ()
@@ -253,6 +66,18 @@ viaTmp a file content = do
 	a tmpfile content
 	renameFile tmpfile file
 
+{- Runs an action with a temp file, then removes the file. -}
+withTempFile :: String -> (FilePath -> Handle -> IO a) -> IO a
+withTempFile template a = bracket create remove use
+	where
+		create = do
+			tmpdir <- catch getTemporaryDirectory (const $ return ".")
+			openTempFile tmpdir template
+		remove (name, handle) = do
+			hClose handle
+			catchBool (removeFile name >> return True)
+		use (name, handle) = a name handle
+
 {- Lists the contents of a directory.
  - Unlike getDirectoryContents, paths are not relative to the directory. -}
 dirContents :: FilePath -> IO [FilePath]
@@ -275,19 +100,23 @@ myHomeDir = do
 catchBool :: IO Bool -> IO Bool
 catchBool = flip catch (const $ return False)
 
-{- when with a monadic conditional -}
-whenM :: Monad m => m Bool -> m () -> m ()
-whenM c a = c >>= flip when a
+{- Return the first value from a list, if any, satisfying the given
+ - predicate -}
+firstM :: (Monad m) => (a -> m Bool) -> [a] -> m (Maybe a)
+firstM _ [] = return Nothing
+firstM p (x:xs) = do
+	q <- p x
+	if q
+		then return (Just x)
+		else firstM p xs
 
-unlessM :: Monad m => m Bool -> m () -> m ()
-unlessM c a = c >>= flip unless a
+{- Returns true if any value in the list satisfies the preducate,
+ - stopping once one is found. -}
+anyM :: (Monad m) => (a -> m Bool) -> [a] -> m Bool
+anyM p = liftM isJust . firstM p
 
-(>>?) :: Monad m => m Bool -> m () -> m ()
-(>>?) = whenM
-
-(>>!) :: Monad m => m Bool -> m () -> m ()
-(>>!) = unlessM
-
--- low fixity allows eg, foo bar >>! error $ "failed " ++ meep
-infixr 0 >>?
-infixr 0 >>!
+{- Checks if a command is available in PATH. -}
+inPath :: String -> IO Bool
+inPath command = getSearchPath >>= anyM indir
+	where
+		indir d = doesFileExist $ d </> command
diff --git a/Utility/Conditional.hs b/Utility/Conditional.hs
new file mode 100644
index 0000000000..85e39ec64c
--- /dev/null
+++ b/Utility/Conditional.hs
@@ -0,0 +1,26 @@
+{- monadic conditional operators
+ -
+ - Copyright 2011 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Conditional where
+
+import Control.Monad (when, unless)
+
+whenM :: Monad m => m Bool -> m () -> m ()
+whenM c a = c >>= flip when a
+
+unlessM :: Monad m => m Bool -> m () -> m ()
+unlessM c a = c >>= flip unless a
+
+(>>?) :: Monad m => m Bool -> m () -> m ()
+(>>?) = whenM
+
+(>>!) :: Monad m => m Bool -> m () -> m ()
+(>>!) = unlessM
+
+-- low fixity allows eg, foo bar >>! error $ "failed " ++ meep
+infixr 0 >>?
+infixr 0 >>!
diff --git a/Utility/CopyFile.hs b/Utility/CopyFile.hs
index 2e06dd92bb..9019357196 100644
--- a/Utility/CopyFile.hs
+++ b/Utility/CopyFile.hs
@@ -9,8 +9,9 @@ module Utility.CopyFile (copyFile) where
 
 import System.Directory (doesFileExist, removeFile)
 
-import Utility
-import qualified SysConfig
+import Utility.Conditional
+import Utility.SafeCommand
+import qualified Build.SysConfig as SysConfig
 
 {- The cp command is used, because I hate reinventing the wheel,
  - and because this allows easy access to features like cp --reflink. -}
diff --git a/Utility/DataUnits.hs b/Utility/DataUnits.hs
index f2bc333ea0..0baa5dd896 100644
--- a/Utility/DataUnits.hs
+++ b/Utility/DataUnits.hs
@@ -137,8 +137,7 @@ compareSizes units abbrev old new
 {- Parses strings like "10 kilobytes" or "0.5tb". -}
 readSize :: [Unit] -> String -> Maybe ByteSize
 readSize units input
-	| null parsednum = Nothing
-	| null parsedunit = Nothing
+	| null parsednum || null parsedunit = Nothing
 	| otherwise = Just $ round $ number * fromIntegral multiplier
 	where
 		(number, rest) = head parsednum
diff --git a/Utility/JSONStream.hs b/Utility/JSONStream.hs
new file mode 100644
index 0000000000..af3766948f
--- /dev/null
+++ b/Utility/JSONStream.hs
@@ -0,0 +1,44 @@
+{- Streaming JSON output.
+ -
+ - Copyright 2011 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.JSONStream (
+	start,
+	add,
+	end
+) where
+
+import Text.JSON
+
+{- Text.JSON does not support building up a larger JSON document piece by
+   piece as a stream. To support streaming, a hack. The JSObject is converted
+   to a string with its final "}" is left off, allowing it to be added to
+   later. -}
+start :: JSON a => [(String, a)] -> String
+start l
+	| last s == endchar = take (length s - 1)  s
+	| otherwise = bad s
+	where
+		s = encodeStrict $ toJSObject l
+
+add :: JSON a => [(String, a)] -> String
+add l
+	| head s == startchar = ',' : drop 1 s
+	| otherwise = bad s
+	where
+		s = start l
+
+end :: String
+end = [endchar, '\n']
+
+startchar :: Char
+startchar = '{'
+
+endchar :: Char
+endchar = '}'
+
+bad :: String -> a
+bad s = error $ "Text.JSON returned unexpected string: " ++ s
diff --git a/Utility/Path.hs b/Utility/Path.hs
new file mode 100644
index 0000000000..9b8041dad0
--- /dev/null
+++ b/Utility/Path.hs
@@ -0,0 +1,92 @@
+{- path manipulation
+ -
+ - Copyright 2010-2011 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Path where
+
+import Data.String.Utils
+import System.Path
+import System.FilePath
+import System.Directory
+import Data.List
+import Data.Maybe
+import Control.Applicative
+
+{- Returns the parent directory of a path. Parent of / is "" -}
+parentDir :: FilePath -> FilePath
+parentDir dir =
+	if not $ null dirs
+	then slash ++ join s (take (length dirs - 1) dirs)
+	else ""
+		where
+			dirs = filter (not . null) $ split s dir
+			slash = if isAbsolute dir then s else ""
+			s = [pathSeparator]
+
+prop_parentDir_basics :: FilePath -> Bool
+prop_parentDir_basics dir
+	| null dir = True
+	| dir == "/" = parentDir dir == ""
+	| otherwise = p /= dir
+	where
+		p = parentDir dir
+
+{- Checks if the first FilePath is, or could be said to contain the second.
+ - For example, "foo/" contains "foo/bar". Also, "foo", "./foo", "foo/" etc
+ - are all equivilant.
+ -}
+dirContains :: FilePath -> FilePath -> Bool
+dirContains a b = a == b || a' == b' || (a'++"/") `isPrefixOf` b'
+	where
+		norm p = fromMaybe "" $ absNormPath p "."
+		a' = norm a
+		b' = norm b
+
+{- Converts a filename into a normalized, absolute path. -}
+absPath :: FilePath -> IO FilePath
+absPath file = do
+	cwd <- getCurrentDirectory
+	return $ absPathFrom cwd file
+
+{- Converts a filename into a normalized, absolute path
+ - from the specified cwd. -}
+absPathFrom :: FilePath -> FilePath -> FilePath
+absPathFrom cwd file = fromMaybe bad $ absNormPath cwd file
+	where
+		bad = error $ "unable to normalize " ++ file
+
+{- Constructs a relative path from the CWD to a file.
+ -
+ - For example, assuming CWD is /tmp/foo/bar:
+ -    relPathCwdToFile "/tmp/foo" == ".."
+ -    relPathCwdToFile "/tmp/foo/bar" == "" 
+ -}
+relPathCwdToFile :: FilePath -> IO FilePath
+relPathCwdToFile f = relPathDirToFile <$> getCurrentDirectory <*> absPath f
+
+{- Constructs a relative path from a directory to a file.
+ -
+ - Both must be absolute, and normalized (eg with absNormpath).
+ -}
+relPathDirToFile :: FilePath -> FilePath -> FilePath
+relPathDirToFile from to = path
+	where
+		s = [pathSeparator]
+		pfrom = split s from
+		pto = split s to
+		common = map fst $ filter same $ zip pfrom pto
+		same (c,d) = c == d
+		uncommon = drop numcommon pto
+		dotdots = replicate (length pfrom - numcommon) ".."
+		numcommon = length common
+		path = join s $ dotdots ++ uncommon
+
+prop_relPathDirToFile_basics :: FilePath -> FilePath -> Bool
+prop_relPathDirToFile_basics from to
+	| from == to = null r
+	| otherwise = not (null r)
+	where
+		r = relPathDirToFile from to 
diff --git a/Utility/RsyncFile.hs b/Utility/RsyncFile.hs
index 6e21ba0632..b6c2267e87 100644
--- a/Utility/RsyncFile.hs
+++ b/Utility/RsyncFile.hs
@@ -9,7 +9,7 @@ module Utility.RsyncFile where
 
 import Data.String.Utils
 
-import Utility
+import Utility.SafeCommand
 
 {- Generates parameters to make rsync use a specified command as its remote
  - shell. -}
diff --git a/Utility/SafeCommand.hs b/Utility/SafeCommand.hs
new file mode 100644
index 0000000000..ba9362603e
--- /dev/null
+++ b/Utility/SafeCommand.hs
@@ -0,0 +1,104 @@
+{- safely running shell commands
+ -
+ - Copyright 2010-2011 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.SafeCommand where
+
+import System.Exit
+import qualified System.Posix.Process
+import System.Posix.Process hiding (executeFile)
+import System.Posix.Signals
+import Data.String.Utils
+import System.Log.Logger
+
+{- A type for parameters passed to a shell command. A command can
+ - be passed either some Params (multiple parameters can be included,
+ - whitespace-separated, or a single Param (for when parameters contain
+ - whitespace), or a File.
+ -}
+data CommandParam = Params String | Param String | File FilePath
+	deriving (Eq, Show, Ord)
+
+{- Used to pass a list of CommandParams to a function that runs
+ - a command and expects Strings. -}
+toCommand :: [CommandParam] -> [String]
+toCommand = (>>= unwrap)
+	where
+		unwrap (Param s) = [s]
+		unwrap (Params s) = filter (not . null) (split " " s)
+		-- Files that start with a dash are modified to avoid
+		-- the command interpreting them as options.
+		unwrap (File s@('-':_)) = ["./" ++ s]
+		unwrap (File s) = [s]
+
+{- Run a system command, and returns True or False
+ - if it succeeded or failed.
+ -
+ - SIGINT(ctrl-c) is allowed to propigate and will terminate the program.
+ -}
+boolSystem :: FilePath -> [CommandParam] -> IO Bool
+boolSystem command params = boolSystemEnv command params Nothing
+
+boolSystemEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO Bool
+boolSystemEnv command params env = do
+	-- Going low-level because all the high-level system functions
+	-- block SIGINT etc. We need to block SIGCHLD, but allow
+	-- SIGINT to do its default program termination.
+	let sigset = addSignal sigCHLD emptySignalSet
+	oldint <- installHandler sigINT Default Nothing
+	oldset <- getSignalMask
+	blockSignals sigset
+	childpid <- forkProcess $ childaction oldint oldset
+	mps <- getProcessStatus True False childpid
+	restoresignals oldint oldset
+	case mps of
+		Just (Exited ExitSuccess) -> return True
+		_ -> return False
+	where
+		restoresignals oldint oldset = do
+			_ <- installHandler sigINT oldint Nothing
+			setSignalMask oldset
+		childaction oldint oldset = do
+			restoresignals oldint oldset
+			executeFile command True (toCommand params) env
+
+{- executeFile with debug logging -}
+executeFile :: FilePath -> Bool -> [String] -> Maybe [(String, String)] -> IO ()
+executeFile c path p e = do
+	debugM "Utility.SafeCommand.executeFile" $
+		"Running: " ++ c ++ " " ++ show p ++ " " ++ maybe "" show e
+	System.Posix.Process.executeFile c path p e
+
+{- Escapes a filename or other parameter to be safely able to be exposed to
+ - the shell. -}
+shellEscape :: String -> String
+shellEscape f = "'" ++ escaped ++ "'"
+	where
+		-- replace ' with '"'"'
+		escaped = join "'\"'\"'" $ split "'" f
+
+{- Unescapes a set of shellEscaped words or filenames. -}
+shellUnEscape :: String -> [String]
+shellUnEscape [] = []
+shellUnEscape s = word : shellUnEscape rest
+	where
+		(word, rest) = findword "" s
+		findword w [] = (w, "")
+		findword w (c:cs)
+			| c == ' ' = (w, cs)
+			| c == '\'' = inquote c w cs
+			| c == '"' = inquote c w cs
+			| otherwise = findword (w++[c]) cs
+		inquote _ w [] = (w, "")
+		inquote q w (c:cs)
+			| c == q = findword w cs
+			| otherwise = inquote q (w++[c]) cs
+
+{- For quickcheck. -}
+prop_idempotent_shellEscape :: String -> Bool
+prop_idempotent_shellEscape s = [s] == (shellUnEscape . shellEscape) s
+prop_idempotent_shellEscape_multiword :: [String] -> Bool
+prop_idempotent_shellEscape_multiword s = s == (shellUnEscape . unwords . map shellEscape) s
diff --git a/Remote/Ssh.hs b/Utility/Ssh.hs
similarity index 97%
rename from Remote/Ssh.hs
rename to Utility/Ssh.hs
index fe4e6dfc1a..05269552c7 100644
--- a/Remote/Ssh.hs
+++ b/Utility/Ssh.hs
@@ -5,12 +5,12 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Remote.Ssh where
+module Utility.Ssh where
 
 import Control.Monad.State (liftIO)
 
 import qualified Git
-import Utility
+import Utility.SafeCommand
 import Types
 import Config
 
diff --git a/StatFS.hsc b/Utility/StatFS.hsc
similarity index 98%
rename from StatFS.hsc
rename to Utility/StatFS.hsc
index feb82aa9af..d3e4a507e5 100644
--- a/StatFS.hsc
+++ b/Utility/StatFS.hsc
@@ -45,7 +45,7 @@
 {-# LANGUAGE CPP, ForeignFunctionInterface, EmptyDataDecls #-}
 
 
-module StatFS ( FileSystemStats(..), getFileSystemStats ) where
+module Utility.StatFS ( FileSystemStats(..), getFileSystemStats ) where
 
 import Foreign
 import Foreign.C.Types
diff --git a/Touch.hsc b/Utility/Touch.hsc
similarity index 99%
rename from Touch.hsc
rename to Utility/Touch.hsc
index dd0c38984e..f27ac31360 100644
--- a/Touch.hsc
+++ b/Utility/Touch.hsc
@@ -7,7 +7,7 @@
 
 {-# LANGUAGE ForeignFunctionInterface #-}
 
-module Touch (
+module Utility.Touch (
 	TimeSpec(..),
 	touchBoth,
 	touch
diff --git a/Utility/Url.hs b/Utility/Url.hs
new file mode 100644
index 0000000000..6ddeecc14f
--- /dev/null
+++ b/Utility/Url.hs
@@ -0,0 +1,79 @@
+{- Url downloading.
+ -
+ - Copyright 2011 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Url (
+	exists,
+	download,
+	get
+) where
+
+import Control.Applicative
+import Control.Monad.State (liftIO)
+import qualified Network.Browser as Browser
+import Network.HTTP
+import Network.URI
+
+import Types
+import Messages
+import Utility.SafeCommand
+import Utility
+
+type URLString = String
+
+{- Checks that an url exists and could be successfully downloaded. -}
+exists :: URLString -> IO Bool
+exists url =
+	case parseURI url of
+		Nothing -> return False
+		Just u -> do
+			r <- request u HEAD
+			case rspCode r of
+				(2,_,_) -> return True
+				_ -> return False
+
+{- Used to download large files, such as the contents of keys.
+ - Uses wget or curl program for its progress bar. (Wget has a better one,
+ - so is preferred.) -}
+download :: URLString -> FilePath -> Annex Bool
+download url file = do
+	showOutput -- make way for program's progress bar
+	e <- liftIO $ inPath "wget"
+	if e
+		then
+			liftIO $ boolSystem "wget"
+				[Params "-c -O", File file, File url]
+		else
+			-- Uses the -# progress display, because the normal
+			-- one is very confusing when resuming, showing
+			-- the remainder to download as the whole file,
+			-- and not indicating how much percent was
+			-- downloaded before the resume.
+			liftIO $ boolSystem "curl" 
+				[Params "-L -C - -# -o", File file, File url]
+
+{- Downloads a small file. -}
+get :: URLString -> IO String
+get url =
+	case parseURI url of
+		Nothing -> error "url parse error"
+		Just u -> do
+			r <- request u GET
+			case rspCode r of
+				(2,_,_) -> return $ rspBody r
+				_ -> error $ rspReason r
+
+{- Makes a http request of an url. For example, HEAD can be used to
+ - check if the url exists, or GET used to get the url content (best for
+ - small urls). -}
+request :: URI -> RequestMethod -> IO (Response String)
+request url requesttype = Browser.browse $ do
+	Browser.setErrHandler ignore
+	Browser.setOutHandler ignore
+	Browser.setAllowRedirects True
+	snd <$> Browser.request (mkRequest requesttype url :: Request_String)
+	where
+		ignore = const $ return ()
diff --git a/Version.hs b/Version.hs
index 7e6910fbe4..fcf6bc4d1d 100644
--- a/Version.hs
+++ b/Version.hs
@@ -7,8 +7,6 @@
 
 module Version where
 
-import Control.Monad (unless)
-
 import Types
 import qualified Annex
 import qualified Git
@@ -39,14 +37,11 @@ getVersion = do
 setVersion :: Annex ()
 setVersion = setConfig versionField defaultVersion
 
-checkVersion :: Annex ()
-checkVersion = getVersion >>= handle
+checkVersion :: Version -> Annex ()
+checkVersion v
+	| v `elem` supportedVersions = return ()
+	| v `elem` upgradableVersions = err "Upgrade this repository: git-annex upgrade"
+	| otherwise = err "Upgrade git-annex."
 	where
-		handle Nothing = error "First run: git-annex init"
-		handle (Just v) = unless (v `elem` supportedVersions) $
-			error $ "Repository version " ++ v ++ 
-				" is not supported. " ++
-				msg v
-		msg v
-			| v `elem` upgradableVersions = "Upgrade this repository: git-annex upgrade"
-			| otherwise = "Upgrade git-annex."
+		err msg = error $ "Repository version " ++ v ++
+			" is not supported. " ++ msg
diff --git a/configure.hs b/configure.hs
index bfdfa32dd5..9f7179c539 100644
--- a/configure.hs
+++ b/configure.hs
@@ -3,7 +3,7 @@
 import System.Directory
 import Data.List
 
-import TestConfig
+import Build.TestConfig
 
 tests :: [TestCase]
 tests =
diff --git a/debian/changelog b/debian/changelog
index 5619f0892e..dc10f82052 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,43 @@
+git-annex (3.20110906) unstable; urgency=low
+
+  * Improve display of newlines around error and warning messages.
+  * Fix Makefile to work with cabal again.
+
+ -- Joey Hess <joeyh@debian.org>  Tue, 06 Sep 2011 13:45:16 -0400
+
+git-annex (3.20110902) unstable; urgency=low
+
+  * Set EMAIL when running test suite so that git does not need to be
+    configured first. Closes: #638998
+  * The wget command will now be used in preference to curl, if available.
+  * init: Make description an optional parameter.
+  * unused, status: Sped up by avoiding unnecessary stats of annexed files.
+  * unused --remote: Reduced memory use to 1/4th what was used before.
+  * Add --json switch, to produce machine-consumable output.
+
+ -- Joey Hess <joeyh@debian.org>  Fri, 02 Sep 2011 21:20:37 -0400
+
+git-annex (3.20110819) unstable; urgency=low
+
+  * Now "git annex init" only has to be run once, when a git repository
+    is first being created. Clones will automatically notice that git-annex
+    is in use and automatically perform a basic initalization. It's
+    still recommended to run "git annex init" in any clones, to describe them.
+  * Added annex-cost-command configuration, which can be used to vary the
+    cost of a remote based on the output of a shell command.
+  * Fix broken upgrade from V1 repository. Closes: #638584
+
+ -- Joey Hess <joeyh@debian.org>  Fri, 19 Aug 2011 20:34:09 -0400
+
+git-annex (3.20110817) unstable; urgency=low
+
+  * Fix shell escaping in rsync special remote.
+  * addurl: --fast can be used to avoid immediately downloading the url.
+  * Added support for getting content from git remotes using http (and https).
+  * Added curl to Debian package dependencies.
+
+ -- Joey Hess <joeyh@debian.org>  Wed, 17 Aug 2011 01:29:02 -0400
+
 git-annex (3.20110719~bpo60+1) squeeze-backports; urgency=low
 
   * Bugfix: Make add ../ work.
diff --git a/debian/control b/debian/control
index bb9118ea0c..6815944eba 100644
--- a/debian/control
+++ b/debian/control
@@ -12,6 +12,7 @@ Build-Depends:
 	libghc6-http-dev,
 	libghc6-utf8-string-dev,
 	libghc6-testpack-dev [any-i386 any-amd64],
+	libghc6-json-dev,
 	ikiwiki,
 	perlmagick,
 	git | git-core,
@@ -29,6 +30,7 @@ Depends: ${misc:Depends}, ${shlibs:Depends},
 	git | git-core,
 	uuid,
 	rsync,
+	wget | curl,
 	openssh-client
 Suggests: graphviz, bup, gnupg
 Description: manage files with git, without checking their contents into git
diff --git a/debian/copyright b/debian/copyright
index 112ad54aaa..a8a38913e4 100644
--- a/debian/copyright
+++ b/debian/copyright
@@ -8,7 +8,7 @@ License: GPL-3+
  this package's source, or in /usr/share/common-licenses/GPL-3 on
  Debian systems.
 
-Files: StatFS.hsc
+Files: Utility/StatFS.hsc
 Copyright: Jose A Ortega Ruiz <jao@gnu.org>
 License: BSD-3-clause
  -- All rights reserved.
diff --git a/doc/backends.mdwn b/doc/backends.mdwn
index 9e698032d8..ebcdedc2a7 100644
--- a/doc/backends.mdwn
+++ b/doc/backends.mdwn
@@ -1,24 +1,16 @@
-git-annex uses a key-value abstraction layer to allow file contents to be
-stored in different ways. In theory, any key-value storage system could be
-used to store file contents.
-
 When a file is annexed, a key is generated from its content and/or metadata.
 The file checked into git symlinks to the key. This key can later be used
 to retrieve the file's content (its value).
 
-Multiple pluggable backends are supported, and a single repository
-can use different backends for different files.
+Multiple pluggable key-value backends are supported, and a single repository
+can use different ones for different files.
 
-These backends can transfer file contents between configured git remotes.
-It's also possible to use [[special_remotes]], such as Amazon S3 with
-these backends.
-
-* `WORM` ("Write Once, Read Many") This backend assumes that any file with
-  the same basename, size, and modification time has the same content. So with
-  this backend, files can be moved around, but should never be added to
-  or changed. This is the default, and the least expensive backend.
-* `SHA1` -- This backend uses a key based on a sha1 checksum. This backend
-  allows modifications of files to be tracked. Its need to generate checksums
+* `WORM` ("Write Once, Read Many") This assumes that any file with
+  the same basename, size, and modification time has the same content.
+  This is the default, and the least expensive backend.
+* `SHA1` -- This uses a key based on a sha1 checksum. This allows
+  verifying that the file content is right, and can avoid duplicates of
+  files with the same content. Its need to generate checksums
   can make it slower for large files.
 * `SHA512`, `SHA384`, `SHA256`, `SHA224` -- Like SHA1, but larger
   checksums. Mostly useful for the very paranoid, or anyone who is
@@ -36,7 +28,7 @@ files, the `.gitattributes` file can be used. The `annex.backend`
 attribute can be set to the name of the backend to use for matching files.
 
 For example, to use the SHA1 backend for sound files, which tend to be
-smallish and might be modified over time, you could set in
+smallish and might be modified or copied over time, you could set in
 `.gitattributes`:
 
 	*.mp3 annex.backend=SHA1
diff --git a/doc/bugs/--git-dir_and_--work-tree_options.mdwn b/doc/bugs/--git-dir_and_--work-tree_options.mdwn
new file mode 100644
index 0000000000..d76a42bfff
--- /dev/null
+++ b/doc/bugs/--git-dir_and_--work-tree_options.mdwn
@@ -0,0 +1,29 @@
+git-annex does not take into account the --git-dir and --work-tree command line options (while they can be useful when scripting).
+
+    > mkdir /tmp/test
+    > cd /tmp/test
+    > git init
+    Initialized empty Git repository in /tmp/test/.git/
+    > git annex init test
+    init test ok
+    > touch foo
+    > cd
+    > git --git-dir=/tmp/test/.git --work-tree=/tmp/test annex add foo
+    git-annex: Not in a git repository.
+
+regular git add works:
+
+    > git --git-dir=/tmp/test/.git --work-tree=/tmp/test add foo
+    > git --git-dir=/tmp/test/.git --work-tree=/tmp/test status 
+    # On branch master
+    #
+    # Initial commit
+    #
+    # Changes to be committed:
+    #   (use "git rm --cached <file>..." to unstage)
+    #
+    #       new file:   foo
+    #
+
+git-annex version: 3.20110702
+
diff --git a/doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn b/doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn
new file mode 100644
index 0000000000..43fb0323c4
--- /dev/null
+++ b/doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn
@@ -0,0 +1,11 @@
+While following the instructions given at the OSX build page , I get this error:
+
+$ make
+ghc -O2 -Wall -ignore-package monads-fd -fspec-constr-count=5 --make git-annex
+
+Utility/JSONStream.hs:14:8:
+    Could not find module `Text.JSON':
+      Use -v to see a list of the files searched for.
+make: *** [git-annex] Error 1
+
+> Updated the instructions. [[done]] --[[Joey]] 
diff --git a/doc/bugs/Cabal_dependency_monadIO_missing.mdwn b/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
new file mode 100644
index 0000000000..b5213b8aa5
--- /dev/null
+++ b/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
@@ -0,0 +1,14 @@
+Just issuing the command `cabal install` results in the following error message.
+
+    Command/Add.hs:54:3:
+        No instance for (Control.Monad.IO.Control.MonadControlIO
+                           (Control.Monad.State.Lazy.StateT Annex.AnnexState IO))
+          arising from a use of `handle' at Command/Add.hs:54:3-24
+
+Adding the dependency for `monadIO` to `git-annex.cabal` should fix this?  
+-- Thomas
+
+> No, it's already satisfied by `monad-control` being listed as a
+> dependency in the cabal file. Your system might be old/new/or broken,
+> perhaps it's time to provide some details about the version of haskell
+> and of `monad-control` you have installed? --[[Joey]] 
diff --git a/doc/bugs/Cabal_dependency_monadIO_missing/comment_1_14be660aa57fadec0d81b32a8b52c66f._comment b/doc/bugs/Cabal_dependency_monadIO_missing/comment_1_14be660aa57fadec0d81b32a8b52c66f._comment
new file mode 100644
index 0000000000..8e38205f00
--- /dev/null
+++ b/doc/bugs/Cabal_dependency_monadIO_missing/comment_1_14be660aa57fadec0d81b32a8b52c66f._comment
@@ -0,0 +1,75 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmFgsNxmnGznb5bbmcoWhoQOoxZZ-io61s"
+ nickname="Thomas"
+ subject="comment 1"
+ date="2011-08-08T09:04:20Z"
+ content="""
+I use Debian Squeeze, I have the Debian package cabal-install 0.8.0-1 installed.
+
+    $ git clone git://git-annex.branchable.com/
+    $ cd git-annex.branchable.com
+    $ cabal update
+    $ cabal install cabal-install
+
+This installed: Cabal-1.10.2.0, zlib-0.5.3.1, cabal-install 0.10.2.
+No version of monad-control or monadIO installed.
+
+    $ ~/.cabal/bin/cabal install
+    Registering QuickCheck-2.4.1.1...
+    Registering Crypto-4.2.3...
+    Registering base-unicode-symbols-0.2.2.1...
+    Registering deepseq-1.1.0.2...
+    Registering hxt-charproperties-9.1.0...
+    Registering hxt-regex-xmlschema-9.0.0...
+    Registering hxt-unicode-9.0.1...
+    Registering hxt-9.1.2...
+    Registering stm-2.2.0.1...
+    Registering hS3-0.5.6...
+    Registering transformers-0.2.2.0...
+    Registering monad-control-0.2.0.1...
+    [1 of 1] Compiling Main             ( Setup.hs, dist/setup/Main.o )
+    Linking ./dist/setup/setup ...
+    ghc -O2 -Wall -ignore-package monads-fd -fspec-constr-count=5 --make configure
+    [1 of 2] Compiling TestConfig       ( TestConfig.hs, TestConfig.o )
+    [2 of 2] Compiling Main             ( configure.hs, configure.o )
+    Linking configure ...
+    ./configure
+      checking version... 3.20110720
+      checking cp -a... yes
+      checking cp -p... yes
+      checking cp --reflink=auto... yes
+      checking uuid generator... uuid
+      checking xargs -0... yes
+      checking rsync... yes
+      checking curl... yes
+      checking bup... yes
+      checking gpg... yes
+      checking sha1... sha1sum
+      checking sha256... sha256sum
+      checking sha512... sha512sum
+      checking sha224... sha224sum
+      checking sha384... sha384sum
+
+    ...
+
+    Command/Add.hs:54:3:
+        No instance for (Control.Monad.IO.Control.MonadControlIO
+                           (Control.Monad.State.Lazy.StateT Annex.AnnexState IO))
+          arising from a use of `handle' at Command/Add.hs:54:3-24
+        Possible fix:
+          add an instance declaration for
+          (Control.Monad.IO.Control.MonadControlIO
+             (Control.Monad.State.Lazy.StateT Annex.AnnexState IO))
+        In the first argument of `($)', namely `handle (undo file key)'
+        In a stmt of a 'do' expression:
+              handle (undo file key) $ moveAnnex key file
+        In the expression:
+            do { handle (undo file key) $ moveAnnex key file;
+                 next $ cleanup file key }
+    cabal: Error: some packages failed to install:
+    git-annex-3.20110719 failed during the building phase. The exception was:
+    ExitFailure 1
+
+After I added a depencency for monadIO to the git-annex.cabal file, it installed correctly.  
+-- Thomas
+"""]]
diff --git a/doc/bugs/Cabal_dependency_monadIO_missing/comment_2_4f4d8e1e00a2a4f7e8a8ab082e16adac._comment b/doc/bugs/Cabal_dependency_monadIO_missing/comment_2_4f4d8e1e00a2a4f7e8a8ab082e16adac._comment
new file mode 100644
index 0000000000..adf7a34e66
--- /dev/null
+++ b/doc/bugs/Cabal_dependency_monadIO_missing/comment_2_4f4d8e1e00a2a4f7e8a8ab082e16adac._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-08-17T04:56:30Z"
+ content="""
+Finally got a chance to try to reproduce this. I followed your recipe exactly in a clean squeeze chroot. monadIO was not installed, but git-annex built ok, using monad-control.
+"""]]
diff --git a/doc/bugs/Prevent_accidental_merges.mdwn b/doc/bugs/Prevent_accidental_merges.mdwn
new file mode 100644
index 0000000000..3e30e02235
--- /dev/null
+++ b/doc/bugs/Prevent_accidental_merges.mdwn
@@ -0,0 +1,14 @@
+With the storage layout v3, pulling the git-annex branch into the master branch is... less than ideal.
+
+The fact that the two branches contain totally different data make an accidental merge worse, arguably.
+
+Adding a tiny binary file called .gitnomerge to both branches would solve that without any noticeable overhead.
+
+Yes, there is an argument to be made that this is too much hand-holding, but I still think it's worth it.
+
+-- Richard
+
+> It should be as easy to undo such an accidential merge
+> as it is to undo any other git commit, right? I quite like that git-annex 
+> no longer adds any clutter to the master branch, and would be reluctant
+> to change that. --[[Joey]]
diff --git a/doc/bugs/add_script-friendly_output_options.mdwn b/doc/bugs/add_script-friendly_output_options.mdwn
new file mode 100644
index 0000000000..7d7bdfc51a
--- /dev/null
+++ b/doc/bugs/add_script-friendly_output_options.mdwn
@@ -0,0 +1,19 @@
+I have a need to use git-annex from a larger program. It'd be great if the information output by some of the commands that is descriptive (for example, whereis) could be sent to stdout in a machine-readable format like (preferably) JSON, or XML. That way I can simply read in the output of the command and use the data directly instead of having to parse it via regexes or other such string manipulation.
+
+This could perhaps be triggered by a --json or --xml flag to the relevant commands.
+
+> This is [[done]], --json is supported by all commands, more or less.
+> 
+> Caveats: 
+> 
+>  * the version, status, and find commands produce custom output and so
+>    no json. This could change for version and status; find needs to just
+>    be a simple list of files, I think 
+>  * The "note" fields may repeat multiple times per object with different
+>    notes and are of course not machine readable, and subject to change.
+>  * Output of helper commands like rsync is not diverted away, and
+>    could clutter up the json output badly. Should only affect commands
+>    that transfer data. And AFAICS, wget and rsync both output their
+>    progress displays to stderr, so shouldn't be a problem.
+> 
+> --[[Joey]] 
diff --git a/doc/bugs/fsck_output.mdwn b/doc/bugs/fsck_output.mdwn
index 90af1600d8..1b00dd7b37 100644
--- a/doc/bugs/fsck_output.mdwn
+++ b/doc/bugs/fsck_output.mdwn
@@ -34,3 +34,13 @@ The newline is in the wrong place and confuses the user. It should be printed _a
 > failed
 > 
 > --[[Joey]] 
+
+>> Well, I fixed this in all cases except a thrown non-IO error (last
+>> example aboce), which output is printed by haskell's runtime. I'd
+>> have to add a second error handler to handle those, and it's not
+>> clear what it would do. Often an error will occur before anything
+>> else is printed, and then the current behavior is right; if something
+>> has been printed it would be nice to have a newline before the error,
+>> but by the time the error is caught we'd be out of the annex monad
+>> and not really have any way to know if something has been printed.
+>> I think my fix is good enough [[done]] --[[Joey]] 
diff --git a/doc/bugs/rsync_special_remote_fails_to___96__get__96___files_which_have_names_containing_spaces.mdwn b/doc/bugs/rsync_special_remote_fails_to___96__get__96___files_which_have_names_containing_spaces.mdwn
new file mode 100644
index 0000000000..040d86bb87
--- /dev/null
+++ b/doc/bugs/rsync_special_remote_fails_to___96__get__96___files_which_have_names_containing_spaces.mdwn
@@ -0,0 +1,50 @@
+    ~$ mkdir test annex
+    ~$ cd test
+    ~$ git init
+    Initialized empty Git repository in /home/user/test/.git/
+    ~$ git annex init test
+    init test ok
+    ~$ git annex initremote localrsync encryption=none type=rsync rsyncurl=localhost:annex/
+    initremote localrsync ok
+    ~$ cp /home/user/Music/Charming\ Hostess/Eat/03\ Mi\ Nuera.ogg ./
+    ~$ git annex add 03\ Mi\ Nuera.ogg
+    add 03 Mi Nuera.ogg ok
+    (Recording state in git...)
+    ~$ git commit -m "add ogg"
+    fatal: No HEAD commit to compare with (yet)
+    fatal: No HEAD commit to compare with (yet)
+    [master (root-commit) 12608af] add ogg
+     1 files changed, 1 insertions(+), 0 deletions(-)
+     create mode 120000 03 Mi Nuera.ogg
+    ~$ git annex move 03\ Mi\ Nuera.ogg --to localrsync
+    move 03 Mi Nuera.ogg (checking localrsync...) (to localrsync...) 
+    sending incremental file list
+    1X/
+    1X/39/
+    1X/39/WORM-s6296772-m1311874383--03 Mi Nuera.ogg/
+    1X/39/WORM-s6296772-m1311874383--03 Mi Nuera.ogg/WORM-s6296772-m1311874383--03 Mi Nuera.ogg
+         6296772 100%   42.98MB/s    0:00:00 (xfer#1, to-check=0/5)
+    
+    sent 6297754 bytes  received 43 bytes  4198531.33 bytes/sec
+    total size is 6296772  speedup is 1.00
+    ok
+    ~$ git annex get 03\ Mi\ Nuera.ogg
+    get 03 Mi Nuera.ogg (from localrsync...) 
+    rsync: link_stat "/home/user/annex/1X/39/WORM-s6296772-m1311874383--03" failed: No such file or directory (2)
+    rsync: link_stat "/home/user/Mi" failed: No such file or directory (2)
+    rsync: change_dir "/home/user/Nuera.ogg" failed: No such file or directory (2)
+    rsync: link_stat "/home/user/Mi" failed: No such file or directory (2)
+    rsync: link_stat "/home/user/Nuera.ogg" failed: No such file or directory (2)
+    
+    sent 8 bytes  received 12 bytes  13.33 bytes/sec
+    total size is 0  speedup is 0.00
+    rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1526) [Receiver=3.0.7]
+    
+      rsync failed -- run git annex again to resume file transfer
+      Unable to access these remotes: localrsync
+      Try making some of these repositories available:
+            b8b1ea7a-b93f-11e0-b712-d7bffb6e61e6  -- localrsync
+    failed
+    git-annex: 1 failed
+
+> [[fixed|done]] --[[Joey]] 
diff --git a/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn b/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn
new file mode 100644
index 0000000000..ba7dcad300
--- /dev/null
+++ b/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn
@@ -0,0 +1,20 @@
+Let's say that http://people.collabora.com/~alsuren/git/fate-suite.git/ is a bare git repo. It has been 'git update-server-info'd so that it can be served on a dumb http server.
+
+The repo is also a git annex remote, created using the following commands:
+
+* git remote add alsuren git+ssh://people.collabora.co.uk/user/alsuren/public_html/fate-suite.git
+* git push alsuren --all
+* git annex copy --to=alsuren
+
+so http://people.collabora.com/~alsuren/git/fate-suite.git/annex is a valid git annex (though listing dirs is forbidden, so you need to know the filenames ahead of time).
+
+I would like to be able to use the following commands to get a clone of the repo:
+
+* git clone http://people.collabora.com/~alsuren/git/fate-suite.git/
+* cd fate-suite
+* git annex get
+
+This would allow contributors to quickly get a copy of our upstream repo and start contributing with minimal bandwidth/effort.
+
+> This is now supported.. I look forward to seeing your project using it!
+> --[[Joey]] [[!tag done]]
diff --git a/doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn b/doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn
new file mode 100644
index 0000000000..2f486ad652
--- /dev/null
+++ b/doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn
@@ -0,0 +1,3 @@
+When the test suite cannot be compiled, the build just fails silenty. This means that in automated builds there is no easy way to ensure that the generated binaries have passed the test suite, because it may not even have been run! IMHO, "make test" should fail (i.e. return a non-zero exit code) when it can't succeeed.
+
+> Ok, fixed. --[[Joey]] [[done]]
diff --git a/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn b/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn
new file mode 100644
index 0000000000..aad119ffd3
--- /dev/null
+++ b/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn
@@ -0,0 +1,36 @@
+as of git-annex version 3.20110719, all git-annex commits only contain the word "update" as a commit message. given that the contents of the commit are pretty non-descriptive (SHA1 hashes for file names, uuids for repository names), i suggest to have more descriptive commit messages, as shown here:
+
+    /mnt/usb_disk/photos/2011$ git annex get
+    /mnt/usb_disk/photos/2011$ git show git-annex
+    [...]
+    usb-disk-photos: get 2011
+    
+    * 10 files retrieved from 2 sources (9 from local-harddisk, 1 from my-server)
+    * 120 files were already present
+    * 2 files could not be retrieved
+    /mnt/usb_disk/photos/2011$ cd ~/photos/2011/07
+    ~/photos/2011/07$ git copy --to my-server
+    ~/photos/2011/07$ git show git-annex
+    [...]
+    local-harddisk: copy 2011/07 to my-server
+    
+    * 20 files pushed
+    ~/photos/2011/07$
+
+in my opinion, the messages should at least contain
+
+* what command was used
+* in which repository they were executed
+* which files or directories they affected (not necessarily all files, but what was given on command line or implicitly from the working directory)
+
+--[[chrysn]]
+
+> The implementation of the git-annex branch precludes more descriptive
+> commit messages, since a single commit can include changes that were
+> previously staged to the branch's index file, or spooled to its journal
+> by other git-annex commands (either concurrently running or
+> interrupted commands, or even changes needed to automatically merge
+> other git-annex branches).
+> 
+> It would be possible to make it *less* verbose, with an empty commit
+> message. :) --[[Joey]] 
diff --git a/doc/cheatsheet.mdwn b/doc/cheatsheet.mdwn
index 9f7c146c82..9ccb22e3e2 100644
--- a/doc/cheatsheet.mdwn
+++ b/doc/cheatsheet.mdwn
@@ -1,4 +1,4 @@
-A suppliment to the [[walkthrough]].
+A supplement to the [[walkthrough]].
 
 [[!toc]]
 
diff --git a/doc/copies.mdwn b/doc/copies.mdwn
index 16eba19c81..93cbd8ea80 100644
--- a/doc/copies.mdwn
+++ b/doc/copies.mdwn
@@ -1,8 +1,8 @@
-The WORM and SHA1 key-value [[backends]] store data inside
-your git repository's `.git` directory, not in some external data store.
+Annexed data is stored inside  your git repository's `.git/annex` directory.
+Some [[special_remotes]] can store annexed data elsewhere.
 
 It's important that data not get lost by an ill-considered `git annex drop`
-command.  So, then using those backends, git-annex can be configured to try
+command.  So, git-annex can be configured to try
 to keep N copies of a file's content available across all repositories. 
 (Although [[untrusted_repositories|trust]] don't count toward this total.)
 
diff --git a/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_2_f5ebb7f43dcef861ecc13373fb1e263f._comment b/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_2_f5ebb7f43dcef861ecc13373fb1e263f._comment
new file mode 100644
index 0000000000..9601003798
--- /dev/null
+++ b/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_2_f5ebb7f43dcef861ecc13373fb1e263f._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmL8pteP2jbYJUn1M3CbeLDvz2SWAA1wtg"
+ nickname="Kristian"
+ subject="Solution"
+ date="2011-07-31T15:24:25Z"
+ content="""
+Yes, it can read id3-tags and guess titles from movie filenames but it sometimes gets confused by the filename metadata provided by the WORM-backend.
+
+I think I have a good enough solution to this problem. It's not efficient when it comes to renames but handles adding and deletion just fine
+
+    rsync -vaL --delete source dest
+
+The -L flag looks at symbolic links and copies the actual data they are pointing to. Of course \"source\" must have all data locally for this to work.
+
+"""]]
diff --git a/doc/forum/advantages_of_SHA__42___over_WORM.mdwn b/doc/forum/advantages_of_SHA__42___over_WORM.mdwn
new file mode 100644
index 0000000000..5b544593f5
--- /dev/null
+++ b/doc/forum/advantages_of_SHA__42___over_WORM.mdwn
@@ -0,0 +1,5 @@
+Thanks for creating git-annex.
+
+I am confused about the advantages of the SHA* backends over WORM.  The "backends" page in this wiki says that with WORM, files "can be moved around, but should never be added to or changed".  But I don't see any difference to SHA* files as long as the premise of WORM that "any file with the same basename, size, and modification time has the same content" is true.  Using "git annex unlock", WORM files can be modified in the same way as SHA* files.
+
+If the storage I use is dependable (i.e. I don't need SHA checksums for detection of corruption), and I don't need to optimize for the case that the modification date of a file is changed but the contents stay the same, and if it is unlikely that several files will be identical, is there actually any advantage in using SHA*?
diff --git a/doc/forum/advantages_of_SHA__42___over_WORM/comment_1_96c354cac4b5ce5cf6664943bc84db1d._comment b/doc/forum/advantages_of_SHA__42___over_WORM/comment_1_96c354cac4b5ce5cf6664943bc84db1d._comment
new file mode 100644
index 0000000000..218027ca53
--- /dev/null
+++ b/doc/forum/advantages_of_SHA__42___over_WORM/comment_1_96c354cac4b5ce5cf6664943bc84db1d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-08-29T16:10:38Z"
+ content="""
+You're right -- as long as nothing changes a file without letting the modification time update, editing WORM files is safe.
+"""]]
diff --git a/doc/forum/version_3_upgrade.mdwn b/doc/forum/version_3_upgrade.mdwn
new file mode 100644
index 0000000000..7fdbcbc805
--- /dev/null
+++ b/doc/forum/version_3_upgrade.mdwn
@@ -0,0 +1,9 @@
+after upgrading to git-annex 3, i'm stuck with diverging git-annex branches -- i didn't manage to follow this line in the directions:
+
+> After this upgrade, you should make sure you include the git-annex branch when git pushing and pulling.
+
+could you explain how to do that in a littel more detail? git pull seems to only merge master, although i have these ``.git/config`` settings:
+
+    [branch "git-annex"]
+    	remote = origin
+    	merge = git-annex
diff --git a/doc/forum/version_3_upgrade/comment_1_05fc9c9cad26c520bebb98c852c71e35._comment b/doc/forum/version_3_upgrade/comment_1_05fc9c9cad26c520bebb98c852c71e35._comment
new file mode 100644
index 0000000000..18746225e9
--- /dev/null
+++ b/doc/forum/version_3_upgrade/comment_1_05fc9c9cad26c520bebb98c852c71e35._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-08-17T01:33:08Z"
+ content="""
+It's ok that `git pull` does not merge the git-annex branch. You can merge it with `git annex merge`, or it will be done
+automatically when you use other git-annex commands.
+
+If you use `git pull` and `git push` without any options, the defaults will make git pull and push the git-annex branch automatically.
+
+But if you're in the habit of doing `git push origin master`, that won't cause the git-annex branch to be pushed (use `git push origin git-annex` to manually push it then). Similarly, `git pull origin master` won't pull it. And also, the `remote.origin.fetch` setting in `.git/config` can be modified in ways that make `git pull` not automatically pull the git-annex branch. So those are the things to avoid after upgrade to v3, basically.
+"""]]
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 11f617f1ba..0a484a3842 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -72,15 +72,15 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * get [path ...]
 
-  Makes the content of annexed files available in this repository. Depending
-  on the backend used, this will involve copying them from another repository,
-  or downloading them, or transferring them from some kind of key-value store.
+  Makes the content of annexed files available in this repository. This
+  will involve copying them from another repository, or downloading them,
+  or transferring them from some kind of key-value store.
 
 * drop [path ...]
 
   Drops the content of annexed files from this repository. 
 
-  git-annex may refuse to drop content if the backend does not think
+  git-annex may refuse to drop content if it does not think
   it is safe to do so, typically because of the setting of annex.numcopies.
 
 * move [path ...]
@@ -119,10 +119,14 @@ Many git-annex commands will stage changes for later `git commit` by you.
   Use this to undo an unlock command if you don't want to modify
   the files, or have made modifications you want to discard.
 
-* init description
+* init [description]
 
-  Initializes git-annex with a description of the git repository,
-  and sets up `.gitattributes` and the pre-commit hook.
+  Until a repository (or one of its remotes) has been initialized,
+  git-annex will refuse to operate on it, to avoid accidentially
+  using it in a repository that was not intended to have an annex.
+
+  It's useful, but not mandatory, to initialize each new clone
+  of a repository with its own description.
 
 * describe repository description
 
@@ -201,14 +205,14 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * migrate [path ...]
 
-  Changes the specified annexed files to store their content in the
-  default backend (or the one specified with --backend). Only files whose
-  content is currently available are migrated.
+  Changes the specified annexed files to use the default key-value backend
+  (or the one specified with --backend). Only files whose content
+  is currently available are migrated.
 
-  Note that the content is not removed from the backend it was previously in.
-  Use `git annex unused` to find and remove such content.
+  Note that the content is also still available using the old key after
+  migration. Use `git annex unused` to find and remove the old key.
 
-  Normally, nothing will be done to files already in the backend.
+  Normally, nothing will be done to files already using the new backend.
   However, if a backend changes the information it uses to construct a key,
   this can also be used to migrate files to use the new key format.
 
@@ -282,10 +286,12 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
   Downloads each url to a file, which is added to the annex.
 
+  To avoid immediately downloading the url, specify --fast
+
 * fromkey file
 
   This plumbing-level command can be used to manually set up a file
-  to link to a specified key in the key-value backend.
+  in the git repository to link to a specified key.
 
 * dropkey [key ...]
 
@@ -331,12 +337,18 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * --quiet
 
-  Avoid the default verbose logging of what is done; only show errors
+  Avoid the default verbose display of what is done; only show errors
   and progress displays.
 
 * --verbose
 
-  Enable verbose logging.
+  Enable verbose display.
+
+* --json
+
+  Rather than the normal output, generate JSON. This is intended to be
+  parsed by programs that use git-annex. Each line of output is a JSON
+  object.
 
 * --debug
 
@@ -416,6 +428,12 @@ Here are all the supported configuration settings.
   The default cost is 100 for local repositories, and 200 for remote
   repositories.
 
+* `remote.<name>.annex-cost-command`
+
+  If set, the command is run, and the number it outputs is used as the cost.
+  This allows varying the cost based on eg, the current network. The
+  cost-command can be any shell command line.
+
 * `remote.<name>.annex-ignore`
 
   If set to `true`, prevents git-annex
@@ -486,8 +504,8 @@ Here are all the supported configuration settings.
 
 # CONFIGURATION VIA .gitattributes
 
-The backend used when adding a new file to the annex can be configured
-on a per-file-type basis via `.gitattributes` files. In the file,
+The key-value backend used when adding a new file to the annex can be
+configured on a per-file-type basis via `.gitattributes` files. In the file,
 the `annex.backend` attribute can be set to the name of the backend to
 use. For example, this here's how to use the WORM backend by default,
 but the SHA1 backend for ogg files:
diff --git a/doc/index.mdwn b/doc/index.mdwn
index 8975c82de7..4b7159cd53 100644
--- a/doc/index.mdwn
+++ b/doc/index.mdwn
@@ -12,7 +12,7 @@ To get a feel for it, see the [[walkthrough]].
 * [[forum]]
 * [[comments]]
 * [[contact]]
-* <a href="http://flattr.com/thing/84843/git-annex"><img src="http://api.flattr.com/button/button-compact-static-100x17.png" alt="Flattr this" title="Flattr this" /></a>
+* <a href="http://flattr.com/thing/84843/git-annex"><img src="https://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" /></a>
 
 [[News]]:
 
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 38963695b8..18962d5d2f 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -27,13 +27,14 @@ To build and use git-annex, you will need:
   * [QuickCheck 2](http://hackage.haskell.org/package/QuickCheck)
   * [HTTP](http://hackage.haskell.org/package/HTTP)
   * [hS3](http://hackage.haskell.org/package/hS3) (optional, but recommended)
+  * [json](http://hackage.haskell.org/package/json)
 * Shell commands
   * [git](http://git-scm.com/)
   * [uuid](http://www.ossp.org/pkg/lib/uuid/)
     (or `uuidgen` from util-linux)
   * [xargs](http://savannah.gnu.org/projects/findutils/)
   * [rsync](http://rsync.samba.org/)
-  * [curl](http://http://curl.haxx.se/) (optional, but recommended)
+  * [wget](http://www.gnu.org/software/wget/) or [curl](http://http://curl.haxx.se/) (optional, but recommended)
   * [sha1sum](ftp://ftp.gnu.org/gnu/coreutils/) (optional, but recommended;
     a sha1 command will also do)
   * [gpg](http://gnupg.org/) (optional; needed for encryption)
diff --git a/doc/install/Debian/comment_1_029486088d098c2d4f1099f2f0e701a9._comment b/doc/install/Debian/comment_1_029486088d098c2d4f1099f2f0e701a9._comment
new file mode 100644
index 0000000000..9a4ed7c31d
--- /dev/null
+++ b/doc/install/Debian/comment_1_029486088d098c2d4f1099f2f0e701a9._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawla7u6eLKNYZ09Z7xwBffqLaXquMQC07fU"
+ nickname="Matthias"
+ subject="squeeze-backports update?"
+ date="2011-08-17T12:34:46Z"
+ content="""
+Is there going to be an update of git-annex in debian squeeze-backports to a version that supports repository version 3?
+Thx
+"""]]
diff --git a/doc/install/Debian/comment_2_648e3467e260cdf233acdb0b53313ce0._comment b/doc/install/Debian/comment_2_648e3467e260cdf233acdb0b53313ce0._comment
new file mode 100644
index 0000000000..b8b3d68f33
--- /dev/null
+++ b/doc/install/Debian/comment_2_648e3467e260cdf233acdb0b53313ce0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="Re: squeeze-backports update?"
+ date="2011-08-17T15:34:29Z"
+ content="""
+Yes, I uploaded it last night.
+"""]]
diff --git a/doc/install/Fedora.mdwn b/doc/install/Fedora.mdwn
index 7814eec940..068d5c111c 100644
--- a/doc/install/Fedora.mdwn
+++ b/doc/install/Fedora.mdwn
@@ -3,20 +3,5 @@ Installation recipe for Fedora 14.
 <pre>
 sudo yum install ghc cabal-install
 sudo cabal update
-sudo cabal install missingh
-sudo cabal install utf8-string
-sudo cabal install pcre-light
-sudo cabal install quickcheck
-sudo cabal install SHA
-sudo cabal install dataenc
-sudo cabal install HTTP
-sudo cabal install hS3
-
-git clone git://git-annex.branchable.com/
-
-cd git-annex
-sudo make   # For some reason you need to use sudo here as otherwise the cabal installed packages doesn't seem to be there...
-sudo install git-annex
+sudo cabal install git-annex
 </pre>
-
-Originally posted by Jon at <https://gist.github.com/997568> --[[Joey]]
diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index ade4fa30e8..680c331ee6 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -1,26 +1,18 @@
+Install Haskel Platform from [[http://hackage.haskell.org/platform/mac.html]]. The version provided by Macports is too old to work with current versions of git-annex. Then execute
+
 <pre>
-sudo port install haskell-platform git-core ossp-uuid md5sha1sum coreutils pcre
-sudo cabal update
-sudo cabal install missingh
-sudo cabal install utf8-string
-sudo cabal install pcre-light
-sudo cabal install quickcheck  
-sudo cabal install SHA
-sudo cabal install dataenc
-sudo cabal install HTTP
-sudo cabal install hS3 # optional
+sudo port install git-core ossp-uuid md5sha1sum coreutils pcre
+
+sudo ln -s /opt/local/include/pcre.h  /usr/include/pcre.h # This is hack that allows pcre-light to find pcre
 
 # optional: this will enable the gnu tools, (to give sha224sum etc..., it does not override the BSD userland)
 export PATH=$PATH:/opt/local/libexec/gnubin
 
-git clone  git://git-annex.branchable.com/
-
-cd git-annex
-make
-sudo make install
+sudo cabal update
+sudo cabal install git-annex
 </pre>
 
-Originally posted by Jon at <https://gist.github.com/671785> --[[Joey]]
+Originally posted by Jon at <https://gist.github.com/671785> --[[Joey]], modified by [[kristianrumberg]]
 
 See also:
 
diff --git a/doc/install/OSX/comment_1_0a1760bf0db1f1ba89bdb4c62032f631._comment b/doc/install/OSX/comment_1_0a1760bf0db1f1ba89bdb4c62032f631._comment
new file mode 100644
index 0000000000..1148a87cab
--- /dev/null
+++ b/doc/install/OSX/comment_1_0a1760bf0db1f1ba89bdb4c62032f631._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://www.schleptet.net/~cfm/"
+ ip="64.30.148.100"
+ subject="comment 1"
+ date="2011-08-30T14:31:36Z"
+ content="""
+You can also use Homebrew instead of MacPorts.  Homebrew's `haskell-platform` is up-to-date, too:
+
+    brew install haskell-platform git ossp-uuid md5sha1sum coreutils pcre
+    ln -s /usr/local/include/pcre.h /usr/include/pcre.h
+
+As of this writing, however, Homebrew's `md5sha1sum` has a broken mirror.  I wound up getting that from MacPorts anyway.
+"""]]
diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index a4ec5c417b..e80ecbac0d 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -22,7 +22,7 @@ deleting or changing the file contents.
 This branch is managed by git-annex, with the contents listed below.
 
 The file `.git/annex/index` is a separate git index file it uses
-to accumlate changes for the git-annex. Also, `.git/annex/journal/` is used
+to accumulate changes for the git-annex. Also, `.git/annex/journal/` is used
 to record changes before they are added to git.
 
 Note that for speed reasons, git-annex assumes only it will modify this
diff --git a/doc/news/version_0.20110610.mdwn b/doc/news/version_0.20110610.mdwn
deleted file mode 100644
index 9ab9e09076..0000000000
--- a/doc/news/version_0.20110610.mdwn
+++ /dev/null
@@ -1,6 +0,0 @@
-git-annex 0.20110610 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Add --numcopies option.
-   * Add --trust, --untrust, and --semitrust options.
-   * get --from is the same as copy --from
-   * Bugfix: Fix fsck to not think all SHAnE keys are bad."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20110624.mdwn b/doc/news/version_3.20110624.mdwn
deleted file mode 100644
index 6204673bd2..0000000000
--- a/doc/news/version_3.20110624.mdwn
+++ /dev/null
@@ -1,33 +0,0 @@
-News for git-annex 3.20110624:
-
-There has been another change to the git-annex data store.
-Use `git annex upgrade` to migrate your repositories to the new
-layout. See [[upgrades]].
-
-The significant change this time is that the .git-annex/ directory
-is gone; instead there is a git-annex branch that is automatically
-maintained by git-annex, and encapsulates all its state nicely out
-of your way.
-
-You should make sure you include the git-annex branch when
-git pushing and pulling.
-
-git-annex 3.20110624 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * New repository format, annex.version=3. Use `git annex upgrade` to migrate.
-   * git-annex now stores its logs in a git-annex branch.
-   * merge: New subcommand. Auto-merges the new git-annex branch.
-   * Improved handling of bare git repos with annexes. Many more commands will
-     work in them.
-   * git-annex is now more robust; it will never leave state files
-     uncommitted when some other git process comes along and locks the index
-     at an inconvenient time.
-   * rsync is now used when copying files from repos on other filesystems.
-     cp is still used when copying file from repos on the same filesystem,
-     since --reflink=auto can make it significantly faster on filesystems
-     such as btrfs.
-   * Allow --trust etc to specify a repository by name, for temporarily
-     trusting repositories that are not configured remotes.
-   * unlock: Made atomic.
-   * git-union-merge: New git subcommand, that does a generic union merge
-     operation, and operates efficiently without touching the working tree."""]]
diff --git a/doc/news/version_3.20110702.mdwn b/doc/news/version_3.20110702.mdwn
deleted file mode 100644
index a5bb925552..0000000000
--- a/doc/news/version_3.20110702.mdwn
+++ /dev/null
@@ -1,22 +0,0 @@
-News for git-annex 3.20110702:
-
-The URL backend has been removed. Instead the new web remote can be used.
-
-git-annex 3.20110702 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Now the web can be used as a special remote.
-     This feature replaces the old URL backend.
-   * addurl: New command to download an url and store it in the annex.
-   * Sped back up fsck, copy --from, and other commands that often
-     have to read a lot of information from the git-annex branch. Such
-     commands are now faster than they were before introduction of the
-     git-annex branch.
-   * Always ensure git-annex branch exists.
-   * Modify location log parser to allow future expansion.
-   * --force will cause add, etc, to operate on ignored files.
-   * Avoid mangling encoding when storing the description of repository
-     and other content.
-   * cabal can now be used to build git-annex. This is substantially
-     slower than using make, does not build or install documentation,
-     does not run the test suite, and is not particularly recommended,
-     but could be useful to some."""]]
diff --git a/doc/news/version_3.20110705.mdwn b/doc/news/version_3.20110705.mdwn
deleted file mode 100644
index bb4665c047..0000000000
--- a/doc/news/version_3.20110705.mdwn
+++ /dev/null
@@ -1,9 +0,0 @@
-git-annex 3.20110705 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * uninit: Delete the git-annex branch and .git/annex/
-   * unannex: In --fast mode, file content is left in the annex, and a
-     hard link made to it.
-   * uninit: Use unannex in --fast mode, to support unannexing multiple
-     files that link to the same content.
-   * Drop the dependency on the haskell curl bindings, use regular haskell HTTP.
-   * Fix a pipeline stall when upgrading (caused by #624389)."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20110719.mdwn b/doc/news/version_3.20110719.mdwn
new file mode 100644
index 0000000000..5beae32270
--- /dev/null
+++ b/doc/news/version_3.20110719.mdwn
@@ -0,0 +1,7 @@
+git-annex 3.20110719 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * add: Be even more robust to avoid ever leaving the file seemingly deleted.
+     Closes: #[634233](http://bugs.debian.org/634233)
+   * Bugfix: Make add ../ work.
+   * Support the standard git -c name=value
+   * unannex: Clean up use of git commit -a."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20110817.mdwn b/doc/news/version_3.20110817.mdwn
new file mode 100644
index 0000000000..51388f3c78
--- /dev/null
+++ b/doc/news/version_3.20110817.mdwn
@@ -0,0 +1,6 @@
+git-annex 3.20110817 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Fix shell escaping in rsync special remote.
+   * addurl: --fast can be used to avoid immediately downloading the url.
+   * Added support for getting content from git remotes using http (and https).
+   * Added curl to Debian package dependencies."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20110819.mdwn b/doc/news/version_3.20110819.mdwn
new file mode 100644
index 0000000000..bbc6abdbc7
--- /dev/null
+++ b/doc/news/version_3.20110819.mdwn
@@ -0,0 +1,9 @@
+git-annex 3.20110819 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Now "git annex init" only has to be run once, when a git repository
+     is first being created. Clones will automatically notice that git-annex
+     is in use and automatically perform a basic initalization. It's
+     still recommended to run "git annex init" in any clones, to describe them.
+   * Added annex-cost-command configuration, which can be used to vary the
+     cost of a remote based on the output of a shell command.
+   * Fix broken upgrade from V1 repository. Closes: #[638584](http://bugs.debian.org/638584)"""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20110902.mdwn b/doc/news/version_3.20110902.mdwn
new file mode 100644
index 0000000000..e354874ea3
--- /dev/null
+++ b/doc/news/version_3.20110902.mdwn
@@ -0,0 +1,9 @@
+git-annex 3.20110902 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Set EMAIL when running test suite so that git does not need to be
+     configured first. Closes: #[638998](http://bugs.debian.org/638998)
+   * The wget command will now be used in preference to curl, if available.
+   * init: Make description an optional parameter.
+   * unused, status: Sped up by avoiding unnecessary stats of annexed files.
+   * unused --remote: Reduced memory use to 1/4th what was used before.
+   * Add --json switch, to produce machine-consumable output."""]]
\ No newline at end of file
diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index afc6a2cf23..55cd1f1a0f 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -1,6 +1,7 @@
 Most [[backends]] can transfer data to and from configured git remotes.
-Normally those remotes are normal git repositories (bare and non-bare),
-that store the file contents in their own git annex directory.
+Normally those remotes are normal git repositories (bare and non-bare;
+local and remote), that store the file contents in their own git annex
+directory.
 
 But, git-annex also extends git's concept of remotes, with these special
 types of remotes. These can be used just like any normal remote by git-annex.
diff --git a/doc/special_remotes/web.mdwn b/doc/special_remotes/web.mdwn
index 68df31ef4d..a969fb071d 100644
--- a/doc/special_remotes/web.mdwn
+++ b/doc/special_remotes/web.mdwn
@@ -1,7 +1,11 @@
 git-annex can use the WWW as a special remote, downloading urls to files.
-See [[walkthrough/using_web_web]] for usage examples.
+See [[walkthrough/using_the_web]] for usage examples.
 
 ## notes
 
 Currently git-annex only supports downloading content from the web; 
 it cannot upload to it or remove content.
+
+This special remote uses arbitrary urls on the web as the source for content.
+git-annex can also download content from a normal git remote, accessible by
+http.
diff --git a/doc/todo/git-annex_unused_eats_memory.mdwn b/doc/todo/git-annex_unused_eats_memory.mdwn
index 6ce7140045..fcb09a1af7 100644
--- a/doc/todo/git-annex_unused_eats_memory.mdwn
+++ b/doc/todo/git-annex_unused_eats_memory.mdwn
@@ -17,9 +17,3 @@ currently present in the repository (possibly using a bloom filter again),
 and that would yield a shortlist of keys that are probably not used. 
 Then scan thru all files in the repo to make sure that none point to keys
 on the shortlist.
-
-----
-
-`git annex unused --from remote` is much worse, using hundreds of mb of
-memory. It has not been profiled at all yet, and can probably be improved
-somewhat by fixing whatever memory leak it (probably) has.
diff --git a/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn b/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn
index 9ca61bff55..be7e2dacc8 100644
--- a/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn
+++ b/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn
@@ -9,3 +9,5 @@ I'm not sure that the above suggestion is going down a path that really
 makes sense. If you want a list of repository UUIDs and descriptions,
 it's there in machine-usable form in `.git-annex/uuid.log`, there is no
 need to try to pull this info out of git commit messages. --[[Joey]]
+
+[[done]]
diff --git a/doc/todo/smudge.mdwn b/doc/todo/smudge.mdwn
index c51662b282..55de5f5789 100644
--- a/doc/todo/smudge.mdwn
+++ b/doc/todo/smudge.mdwn
@@ -1,5 +1,18 @@
 git-annex should use smudge/clean filters.
 
+----
+
+Update: Currently, this does not look likely to work. In particular,
+the clean filter needs to consume all stdin from git, which consists of the
+entire content of the file. It cannot optimise by directly accessing
+the file in the repository, because git may be cleaning a different
+version of the file during a merge. 
+
+So every `git status` would need to read the entire content of all
+available files, and checksum them, which is too expensive.
+
+----
+
 The clean filter is run when files are staged for commit. So a user could copy
 any file into the annex, git add it, and git-annex's clean filter causes
 the file's key to be staged, while its value is added to the annex.
@@ -8,7 +21,7 @@ The smudge filter is run when files are checked out. Since git annex
 repos have partial content, this would not git annex get the file content.
 Instead, if the content is not currently available, it would need to do
 something like return empty file content. (Sadly, it cannot create a
-symlink, as git still wants to write the file afterwards.
+symlink, as git still wants to write the file afterwards.)
 
 So the nice current behavior of unavailable files being clearly missing due
 to dangling symlinks, would be lost when using smudge/clean filters.
@@ -19,23 +32,68 @@ add` files, and just being able to use `git add` or `git commit -a`,
 and have it use git-annex when .gitattributes says to. Also, annexed
 files can be directly modified without having to `git annex unlock`.
 
-### efficiency
+### design
+
+In .gitattributes, the user would put something like "* filter=git-annex".
+This way they could control which files are annexed vs added normally.
+
+(git-annex could have further controls to allow eg, passing small files
+through to regular processing. At least .gitattributes is a special case,
+it should never be annexed...)
+
+For files not configured this way, git-annex could continue to use
+its symlink method -- this would preserve backwards compatability,
+and even allow mixing the two methods in a repo as desired.
+
+To find files in the repository that are annexed, git-annex would do
+`ls-files` as now, but would check if found files have the appropriate
+filter, rather than the current symlink checks. To determine the key
+of a file, rather than reading its symlink, git-annex would need to
+look up the git blob associated with the file -- this can be done
+efficiently using the existing code in `Branch.catFile`.
+
+The clean filter would inject the file's content into the annex, and hard
+link from the annex to the file. Avoiding duplication of data.
+
+The smudge filter can't do that, so to avoid duplication of data, it
+might always create an empty file. To get the content, `git annex get`
+could be used (which would hard link it). A `post-checkout` hook might
+be used to set up hard links for all currently available content.
+
+#### clean
 
 The trick is doing it efficiently. Since git a2b665d, v1.7.4.1,
 something like this works to provide a filename to the clean script:
 
 	git config --global filter.huge.clean huge-clean %f
 
-This avoids it needing to read all the current file content from stdin
+This could avoid it needing to read all the current file content from stdin
 when doing eg, a git status or git commit. Instead it is passed the
 filename that git is operating on, in the working directory.
+(Update: No, doesn't work; git may be cleaning a different file content
+than is currently on disk, and git requires all stdin be consumed too.)
 
 So, WORM could just look at that file and easily tell if it is one
 it already knows (same mtime and size). If so, it can short-circuit and
 do nothing, file content is already cached.
 
 SHA1 has a harder job. Would not want to re-sha1 the file every time,
-probably. So it'd need a cache of file stat info, mapped to known objects.
+probably. So it'd need a local cache of file stat info, mapped to known
+objects.
+
+But: Even with %f, git actually passes the full file content to the clean
+filter, and if it fails to consume it all, it will crash (may only happen
+if the file is larger than some chunk size; tried with 500 mb file and 
+saw a SIGPIPE.) This means unnecessary works needs to be done, 
+and it slows down *everything*, from `git status` to `git commit`.
+**showstopper** I have sent a patch to the git mailing list to address
+this. <http://marc.info/?l=git&m=131465033512157&w=2> (Update: apparently
+can't be fixed.)
+
+#### smudge
+
+The smudge script can also be provided a filename with %f, but it
+cannot directly write to the file or git gets unhappy.
 
 ### dealing with partial content availability
 
@@ -58,12 +116,13 @@ huge-smudge:
 
 <pre>
 #!/bin/sh
-read sha1
-echo "smudging $sha1" >&2
-if [ -e ~/$sha1 ]; then
-	cat ~/$sha1
+read f
+file="$1"
+echo "smudging $f" >&2
+if [ -e ~/$f ]; then
+	cat ~/$f # possibly expensive copy here
 else
-	echo "$sha1 not available"
+	echo "$f not available"
 fi
 </pre>
 
@@ -71,17 +130,17 @@ huge-clean:
 
 <pre>
 #!/bin/sh
-cat >temp
-if grep -q 'not available' temp; then
-	awk '{print $1}' temp # provide what we would if the content were avail!
-	rm temp
+file="$1"
+cat >/tmp/file
+# in real life, this should be done more efficiently, not trying to read
+# the whole file content!
+if grep -q 'not available' /tmp/file; then
+	awk '{print $1}' /tmp/file # provide what we would if the content were avail!
 	exit 0
 fi
-sha1=`sha1sum temp | cut -d' ' -f1`
-echo "cleaning $sha1" >&2
-ls -l temp >&2
-mv temp ~/$sha1
-echo $sha1
+echo "cleaning $file" >&2
+# XXX store file content here
+echo $file
 </pre>
 
 .gitattributes:
@@ -94,6 +153,6 @@ in .git/config:
 
 <pre>
 [filter "huge"]
-        clean = huge-clean
-        smudge = huge-smudge
+        clean = huge-clean %f
+        smudge = huge-smudge %f
 <pre>
diff --git a/doc/transferring_data.mdwn b/doc/transferring_data.mdwn
index f6ae9bfcde..57873f6f0e 100644
--- a/doc/transferring_data.mdwn
+++ b/doc/transferring_data.mdwn
@@ -1,8 +1,12 @@
 git-annex can transfer data to or from any of a repository's git remotes.
 Depending on where the remote is, the data transfer is done using rsync
-(over ssh, with automatic resume), or plain cp (with copy-on-write
-optimisations on supported filesystems). Some [[special_remotes]]
-are also supported that are not traditional git remotes.
+(over ssh or locally), or plain cp (with copy-on-write
+optimisations on supported filesystems), or using curl (for repositories
+on the web). Some [[special_remotes]] are also supported that are not
+traditional git remotes.
+
+If a data transfer is interrupted, git-annex retains the partial transfer
+to allow it to be automatically resumed later.
 
 It's equally easy to transfer a single file to or from a repository,
 or to launch a retrievel of a massive pile of files from whatever
diff --git a/doc/walkthrough/fsck:_verifying_your_data.mdwn b/doc/walkthrough/fsck:_verifying_your_data.mdwn
index 7e05469a12..d036332fb3 100644
--- a/doc/walkthrough/fsck:_verifying_your_data.mdwn
+++ b/doc/walkthrough/fsck:_verifying_your_data.mdwn
@@ -1,8 +1,8 @@
-You can use the fsck subcommand to check for problems in your data.
-What can be checked depends on the [[backend|backends]] you've used to store
-the data. For example, when you use the SHA1 backend, fsck will verify that
-the checksums of your files are good. Fsck also checks that the annex.numcopies
-setting is satisfied for all files.
+You can use the fsck subcommand to check for problems in your data. What
+can be checked depends on the key-value [[backend|backends]] you've used
+for the data. For example, when you use the SHA1 backend, fsck will verify
+that the checksums of your files are good. Fsck also checks that the
+annex.numcopies setting is satisfied for all files.
 
 	# git annex fsck
 	fsck some_file (checksum...) ok
diff --git a/doc/walkthrough/removing_files.mdwn b/doc/walkthrough/removing_files.mdwn
index 85a7d50a6b..0df6e82a1f 100644
--- a/doc/walkthrough/removing_files.mdwn
+++ b/doc/walkthrough/removing_files.mdwn
@@ -3,4 +3,3 @@ has the file before removing it.
 
 	# git annex drop iso/debian.iso
 	drop iso/Debian_5.0.iso ok
-	# git commit -a -m "freed up space"
diff --git a/doc/walkthrough/removing_files/comment_1_cb65e7c510b75be1c51f655b058667c6._comment b/doc/walkthrough/removing_files/comment_1_cb65e7c510b75be1c51f655b058667c6._comment
new file mode 100644
index 0000000000..1c8719cecd
--- /dev/null
+++ b/doc/walkthrough/removing_files/comment_1_cb65e7c510b75be1c51f655b058667c6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="DavidEdmondson"
+ subject="Is it necessary to commit after the 'drop'?"
+ date="2011-09-05T15:43:25Z"
+ content="""
+In fact is it possible? Nothing changed as far as git is concerned.
+
+"""]]
diff --git a/doc/walkthrough/removing_files/comment_2_64709ea4558915edd5c8ca4486965b07._comment b/doc/walkthrough/removing_files/comment_2_64709ea4558915edd5c8ca4486965b07._comment
new file mode 100644
index 0000000000..f5fb8dc7f5
--- /dev/null
+++ b/doc/walkthrough/removing_files/comment_2_64709ea4558915edd5c8ca4486965b07._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-09-05T15:59:27Z"
+ content="""
+Good catch. It used to be necessary before there was a git-annex branch, but not now.
+"""]]
diff --git a/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn b/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn
index 936d088f1f..cfb70aaf9a 100644
--- a/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn
+++ b/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn
@@ -15,4 +15,3 @@ it:
 	# sudo mount /media/usb
 	# git annex get video/hackity_hack_and_kaxxt.mov
 	get video/hackity_hack_and_kaxxt.mov (from usbdrive...) ok
-	# git commit -a -m "got a video I want to rewatch on the plane"
diff --git a/doc/walkthrough/unused_data.mdwn b/doc/walkthrough/unused_data.mdwn
index fb84193034..e142b576c0 100644
--- a/doc/walkthrough/unused_data.mdwn
+++ b/doc/walkthrough/unused_data.mdwn
@@ -2,7 +2,7 @@ It's possible for data to accumulate in the annex that no files point to
 anymore. One way it can happen is if you `git rm` a file without 
 first calling `git annex drop`. And, when you modify an annexed file, the old
 content of the file remains in the annex. Another way is when migrating
-between backends.
+between key-value [[backends|backend]].
 
 This might be historical data you want to preserve, so git-annex defaults to
 preserving it. So from time to time, you may want to check for such data and
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 29ac63aea1..1fb928f9da 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -11,7 +11,8 @@ import Data.List
 import qualified Git
 import CmdLine
 import Command
-import Utility
+import Utility.Conditional
+import Utility.SafeCommand
 import Options
 
 import qualified Command.ConfigList
diff --git a/git-annex.cabal b/git-annex.cabal
index d80b298f4c..743a187dce 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110719
+Version: 3.20110906
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess <joey@kitenet.net>
@@ -31,7 +31,7 @@ Executable git-annex
   Build-Depends: haskell98, MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, HTTP,
-   base < 5
+   base < 5, json
 
 Executable git-annex-shell
   Main-Is: git-annex-shell.hs
diff --git a/git-union-merge.hs b/git-union-merge.hs
index e763376077..4e1a932b45 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -20,7 +20,7 @@ usage :: IO a
 usage = error $ "bad parameters\n\n" ++ header
 
 tmpIndex :: Git.Repo -> FilePath
-tmpIndex g = Git.workTree g </> Git.gitDir g </> "index.git-union-merge"
+tmpIndex g = Git.gitDir g </> "index.git-union-merge"
 
 setup :: Git.Repo -> IO ()
 setup g = cleanup g -- idempotency
diff --git a/test.hs b/test.hs
index 2352df36a6..4d751a707b 100644
--- a/test.hs
+++ b/test.hs
@@ -24,11 +24,12 @@ import qualified Data.Map as M
 import System.Path (recurseDir)
 import System.IO.HVFS (SystemFS(..))
 
+import Utility.SafeCommand
+
 import qualified Annex
 import qualified Backend
 import qualified Git
 import qualified Locations
-import qualified Utility
 import qualified Types.Backend
 import qualified Types
 import qualified GitAnnex
@@ -42,6 +43,7 @@ import qualified Command.DropUnused
 import qualified Types.Key
 import qualified Config
 import qualified Crypto
+import qualified Utility.Path
 
 -- for quickcheck
 instance Arbitrary Types.Key.Key where
@@ -72,11 +74,12 @@ quickcheck = TestLabel "quickcheck" $ TestList
 	[ qctest "prop_idempotent_deencode" Git.prop_idempotent_deencode
 	, qctest "prop_idempotent_fileKey" Locations.prop_idempotent_fileKey
 	, qctest "prop_idempotent_key_read_show" Types.Key.prop_idempotent_key_read_show
-	, qctest "prop_idempotent_shellEscape" Utility.prop_idempotent_shellEscape
-	, qctest "prop_idempotent_shellEscape_multiword" Utility.prop_idempotent_shellEscape_multiword
+	, qctest "prop_idempotent_shellEscape" Utility.SafeCommand.prop_idempotent_shellEscape
+	, qctest "prop_idempotent_shellEscape_multiword" Utility.SafeCommand.prop_idempotent_shellEscape_multiword
 	, qctest "prop_idempotent_configEscape" RemoteLog.prop_idempotent_configEscape
-	, qctest "prop_parentDir_basics" Utility.prop_parentDir_basics
-	, qctest "prop_relPathDirToFile_basics" Utility.prop_relPathDirToFile_basics
+	, qctest "prop_parentDir_basics" Utility.Path.prop_parentDir_basics
+
+	, qctest "prop_relPathDirToFile_basics" Utility.Path.prop_relPathDirToFile_basics
 	, qctest "prop_cost_sane" Config.prop_cost_sane
 	, qctest "prop_hmacWithCipher_sane" Crypto.prop_hmacWithCipher_sane
 	]
@@ -117,8 +120,8 @@ test_add = "git-annex add" ~: TestList [basic, sha1dup, subdirs]
 			git_annex "add" ["-q", annexedfile] @? "add failed"
 			annexed_present annexedfile
 			writeFile ingitfile $ content ingitfile
-			Utility.boolSystem "git" [Utility.Param "add", Utility.File ingitfile] @? "git add failed"
-			Utility.boolSystem "git" [Utility.Params "commit -q -a -m commit"] @? "git commit failed"
+			boolSystem "git" [Param "add", File ingitfile] @? "git add failed"
+			boolSystem "git" [Params "commit -q -a -m commit"] @? "git commit failed"
 			git_annex "add" ["-q", ingitfile] @? "add ingitfile should be no-op"
 			unannexed ingitfile
 		sha1dup = TestCase $ intmpclonerepo $ do
@@ -145,7 +148,7 @@ test_setkey = "git-annex setkey/fromkey" ~: TestCase $ inmainrepo $ do
 	let key = show $ fromJust r
 	git_annex "setkey" ["-q", "--key", key, tmp] @? "setkey failed"
 	git_annex "fromkey" ["-q", "--key", key, sha1annexedfile] @? "fromkey failed"
-	Utility.boolSystem "git" [Utility.Params "commit -q -a -m commit"] @? "git commit failed"
+	boolSystem "git" [Params "commit -q -a -m commit"] @? "git commit failed"
 	annexed_present sha1annexedfile
 	where
 		tmp = "tmpfile"
@@ -172,7 +175,7 @@ test_drop = "git-annex drop" ~: TestList [noremote, withremote, untrustedremote]
 	where
 		noremote = "no remotes" ~: TestCase $ intmpclonerepo $ do
 			git_annex "get" ["-q", annexedfile] @? "get failed"
-			Utility.boolSystem "git" [Utility.Params "remote rm origin"]
+			boolSystem "git" [Params "remote rm origin"]
 				@? "git remote rm origin failed"
 			r <- git_annex "drop" ["-q", annexedfile]
 			not r @? "drop wrongly succeeded with no known copy of file"
@@ -303,12 +306,12 @@ test_edit = "git-annex edit/commit" ~: TestList [t False, t True]
 			then do
 				-- pre-commit depends on the file being
 				-- staged, normally git commit does this
-				Utility.boolSystem "git" [Utility.Param "add", Utility.File annexedfile]
+				boolSystem "git" [Param "add", File annexedfile]
 					@? "git add of edited file failed"
 				git_annex "pre-commit" ["-q"]
 					@? "pre-commit failed"
 			else do
-				Utility.boolSystem "git" [Utility.Params "commit -q -a -m contentchanged"]
+				boolSystem "git" [Params "commit -q -a -m contentchanged"]
 					@? "git commit of edited file failed"
 		runchecks [checklink, checkunwritable] annexedfile
 		c <- readFile annexedfile
@@ -326,7 +329,7 @@ test_fix = "git-annex fix" ~: intmpclonerepo $ do
 	git_annex "fix" ["-q", annexedfile] @? "fix of present file failed"
 	annexed_present annexedfile
 	createDirectory subdir
-	Utility.boolSystem "git" [Utility.Param "mv", Utility.File annexedfile, Utility.File subdir]
+	boolSystem "git" [Param "mv", File annexedfile, File subdir]
 		@? "git mv failed"
 	git_annex "fix" ["-q", newfile] @? "fix of moved file failed"
 	runchecks [checklink, checkunwritable] newfile
@@ -364,9 +367,9 @@ test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withrem
 	where
 		basicfsck = TestCase $ intmpclonerepo $ do
 			git_annex "fsck" ["-q"] @? "fsck failed"
-			Utility.boolSystem "git" [Utility.Params "config annex.numcopies 2"] @? "git config failed"
+			boolSystem "git" [Params "config annex.numcopies 2"] @? "git config failed"
 			fsck_should_fail "numcopies unsatisfied"
-			Utility.boolSystem "git" [Utility.Params "config annex.numcopies 1"] @? "git config failed"
+			boolSystem "git" [Params "config annex.numcopies 1"] @? "git config failed"
 			corrupt annexedfile
 			corrupt sha1annexedfile
 		withlocaluntrusted = TestCase $ intmpclonerepo $ do
@@ -377,7 +380,7 @@ test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withrem
 			git_annex "trust" ["-q", "."] @? "trust of current repo failed"
 			git_annex "fsck" ["-q", annexedfile] @? "fsck failed on file present in trusted repo"
 		withremoteuntrusted = TestCase $ intmpclonerepo $ do
-			Utility.boolSystem "git" [Utility.Params "config annex.numcopies 2"] @? "git config failed"
+			boolSystem "git" [Params "config annex.numcopies 2"] @? "git config failed"
 			git_annex "get" ["-q", annexedfile] @? "get failed"
 			git_annex "get" ["-q", sha1annexedfile] @? "get failed"
 			git_annex "fsck" ["-q"] @? "fsck failed with numcopies=2 and 2 copies"
@@ -448,9 +451,9 @@ test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do
 	git_annex "get" ["-q", annexedfile] @? "get of file failed"
 	git_annex "get" ["-q", sha1annexedfile] @? "get of file failed"
 	checkunused []
-	Utility.boolSystem "git" [Utility.Params "rm -q", Utility.File annexedfile] @? "git rm failed"
+	boolSystem "git" [Params "rm -q", File annexedfile] @? "git rm failed"
 	checkunused [annexedfilekey]
-	Utility.boolSystem "git" [Utility.Params "rm -q", Utility.File sha1annexedfile] @? "git rm failed"
+	boolSystem "git" [Params "rm -q", File sha1annexedfile] @? "git rm failed"
 	checkunused [annexedfilekey, sha1annexedfilekey]
 
 	-- good opportunity to test dropkey also
@@ -526,10 +529,10 @@ setuprepo :: FilePath -> IO FilePath
 setuprepo dir = do
 	cleanup dir
 	ensuretmpdir
-	Utility.boolSystem "git" [Utility.Params "init -q", Utility.File dir] @? "git init failed"
+	boolSystem "git" [Params "init -q", File dir] @? "git init failed"
 	indir dir $ do
-		Utility.boolSystem "git" [Utility.Params "config user.name", Utility.Param "Test User"] @? "git config failed"
-		Utility.boolSystem "git" [Utility.Params "config user.email test@example.com"] @? "git config failed"
+		boolSystem "git" [Params "config user.name", Param "Test User"] @? "git config failed"
+		boolSystem "git" [Params "config user.email test@example.com"] @? "git config failed"
 	return dir
 
 -- clones are always done as local clones; we cannot test ssh clones
@@ -537,7 +540,7 @@ clonerepo :: FilePath -> FilePath -> IO FilePath
 clonerepo old new = do
 	cleanup new
 	ensuretmpdir
-	Utility.boolSystem "git" [Utility.Params "clone -q", Utility.File old, Utility.File new] @? "git clone failed"
+	boolSystem "git" [Params "clone -q", File old, File new] @? "git clone failed"
 	indir new $ git_annex "init" ["-q", new] @? "git annex init failed"
 	return new
 	
@@ -643,6 +646,9 @@ prepare = do
 	p <- getEnvDefault  "PATH" ""
 	setEnv "PATH" (cwd ++ ":" ++ p) True
 	setEnv "TOPDIR" cwd True
+	-- Avoid git complaining if it cannot determine the user's email
+	-- address.
+	setEnv "EMAIL" "git-annex test <test@example.com>" True
 
 changeToTmpDir :: FilePath -> IO ()
 changeToTmpDir t = do